src/PVE/HA/LRM.pm

   1 package PVE::HA::LRM;
   2
   3 # Local Resource Manager
   4
   5 use strict;
   6 use warnings;
   7 use Data::Dumper;
   8 use POSIX qw(:sys_wait_h);
   9
  10 use PVE::SafeSyslog;
  11 use PVE::Tools;
  12 use PVE::HA::Tools;
  13
  14 # Server can have several states:
  15
  16 my $valid_states = {
  17     wait_for_agent_lock => "waiting for agent lock",
  18     active => "got agent_lock",
  19     lost_agent_lock => "lost agent_lock",
  20 };
  21
  22 sub new {
  23     my ($this, $haenv) = @_;
  24
  25     my $class = ref($this) || $this;
  26
  27     my $self = bless {
  28         haenv => $haenv,
  29         status => { state => 'startup' },
  30         workers => {},
  31         results => {},
  32         shutdown_request => 0,
  33         # mode can be: active, reboot, shutdown, restart
  34         mode => 'active',
  35     }, $class;
  36
  37     $self->set_local_status({ state =>  'wait_for_agent_lock' });
  38
  39     return $self;
  40 }
  41
  42 sub shutdown_request {
  43     my ($self) = @_;
  44
  45     $self->{shutdown_request} = 1;
  46
  47     $self->{mode} = 'restart'; # fixme: detect shutdown/reboot
  48
  49     eval { $self->update_lrm_status(); };
  50     if (my $err = $@) {
  51         $self->log('err', "unable to update lrm status file - $err");
  52     }
  53 }
  54
  55 sub get_local_status {
  56     my ($self) = @_;
  57
  58     return $self->{status};
  59 }
  60
  61 sub set_local_status {
  62     my ($self, $new) = @_;
  63
  64     die "invalid state '$new->{state}'" if !$valid_states->{$new->{state}};
  65
  66     my $haenv = $self->{haenv};
  67
  68     my $old = $self->{status};
  69
  70     # important: only update if if really changed
  71     return if $old->{state} eq $new->{state};
  72
  73     $haenv->log('info', "status change $old->{state} => $new->{state}");
  74
  75     $new->{state_change_time} = $haenv->get_time();
  76
  77     $self->{status} = $new;
  78 }
  79
  80 sub update_lrm_status {
  81     my ($self) = @_;
  82
  83     my $haenv = $self->{haenv};
  84
  85     my $lrm_status = {
  86         mode => $self->{mode},
  87         results => $self->{results},
  88         timestamp => $haenv->get_time(),
  89     };
  90
  91     eval { $haenv->write_lrm_status($lrm_status); };
  92     if (my $err = $@) {
  93         $haenv->log('err', "unable to write lrm status file - $err");
  94         return 0;
  95     }
  96
  97     return 1;
  98 }
  99
 100 sub get_protected_ha_agent_lock {
 101     my ($self) = @_;
 102
 103     my $haenv = $self->{haenv};
 104
 105     my $count = 0;
 106     my $starttime = $haenv->get_time();
 107
 108     for (;;) {
 109
 110         if ($haenv->get_ha_agent_lock()) {
 111             if ($self->{ha_agent_wd}) {
 112                 $haenv->watchdog_update($self->{ha_agent_wd});
 113             } else {
 114                 my $wfh = $haenv->watchdog_open();
 115                 $self->{ha_agent_wd} = $wfh;
 116             }
 117             return 1;
 118         }
 119
 120         last if ++$count > 5; # try max 5 time
 121
 122         my $delay = $haenv->get_time() - $starttime;
 123         last if $delay > 5; # for max 5 seconds
 124
 125         $haenv->sleep(1);
 126     }
 127
 128     return 0;
 129 }
 130
 131 sub active_service_count {
 132     my ($self) = @_;
 133
 134     my $haenv = $self->{haenv};
 135
 136     my $nodename = $haenv->nodename();
 137
 138     my $ss = $self->{service_status};
 139
 140     my $count = 0;
 141
 142     foreach my $sid (keys %$ss) {
 143         my $sd = $ss->{$sid};
 144         next if !$sd->{node};
 145         next if $sd->{node} ne $nodename;
 146         my $req_state = $sd->{state};
 147         next if !defined($req_state);
 148         next if $req_state eq 'stopped';
 149         next if $req_state eq 'freeze';
 150
 151         $count++;
 152     }
 153
 154     return $count;
 155 }
 156
 157 my $wrote_lrm_status_at_startup = 0;
 158
 159 sub do_one_iteration {
 160     my ($self) = @_;
 161
 162     my $haenv = $self->{haenv};
 163
 164     if (!$wrote_lrm_status_at_startup) {
 165         if ($haenv->quorate() && $self->update_lrm_status()) {
 166             $wrote_lrm_status_at_startup = 1;
 167         } else {
 168             # do nothing
 169             $haenv->sleep(5);
 170             return $self->{shutdown_request} ? 0 : 1;
 171         }
 172     }
 173
 174     my $status = $self->get_local_status();
 175     my $state = $status->{state};
 176
 177     my $ms = $haenv->read_manager_status();
 178     $self->{service_status} =  $ms->{service_status} || {};
 179
 180     my $fence_request = PVE::HA::Tools::count_fenced_services($self->{service_status}, $haenv->nodename());
 181
 182     # do state changes first
 183
 184     my $ctime = $haenv->get_time();
 185
 186     if ($state eq 'wait_for_agent_lock') {
 187
 188         my $service_count = $self->active_service_count();
 189
 190         if (!$fence_request && $service_count && $haenv->quorate()) {
 191             if ($self->get_protected_ha_agent_lock()) {
 192                 $self->set_local_status({ state => 'active' });
 193             }
 194         }
 195
 196     } elsif ($state eq 'lost_agent_lock') {
 197
 198         if (!$fence_request && $haenv->quorate()) {
 199             if ($self->get_protected_ha_agent_lock()) {
 200                 $self->set_local_status({ state => 'active' });
 201             }
 202         }
 203
 204     } elsif ($state eq 'active') {
 205
 206         if ($fence_request) {
 207             $haenv->log('err', "node need to be fenced - releasing agent_lock\n");
 208             $self->set_local_status({ state => 'lost_agent_lock'});
 209         } elsif (!$self->get_protected_ha_agent_lock()) {
 210             $self->set_local_status({ state => 'lost_agent_lock'});
 211         }
 212     }
 213
 214     $status = $self->get_local_status();
 215     $state = $status->{state};
 216
 217     # do work
 218
 219     if ($state eq 'wait_for_agent_lock') {
 220
 221         return 0 if $self->{shutdown_request};
 222
 223         $haenv->sleep(5);
 224
 225     } elsif ($state eq 'active') {
 226
 227         my $startime = $haenv->get_time();
 228
 229         my $max_time = 10;
 230
 231         my $shutdown = 0;
 232
 233         # do work (max_time seconds)
 234         eval {
 235             # fixme: set alert timer
 236
 237             if ($self->{shutdown_request}) {
 238
 239                 # fixme: request service stop or relocate ?
 240
 241                 my $service_count = $self->active_service_count();
 242
 243                 if ($service_count == 0) {
 244
 245                     if ($self->{ha_agent_wd}) {
 246                         $haenv->watchdog_close($self->{ha_agent_wd});
 247                         delete $self->{ha_agent_wd};
 248                     }
 249
 250                     $shutdown = 1;
 251                 }
 252             } else {
 253
 254                 $self->manage_resources();
 255
 256             }
 257         };
 258         if (my $err = $@) {
 259             $haenv->log('err', "got unexpected error - $err");
 260         }
 261
 262         return 0 if $shutdown;
 263
 264         $haenv->sleep_until($startime + $max_time);
 265
 266     } elsif ($state eq 'lost_agent_lock') {
 267
 268         # Note: watchdog is active an will triger soon!
 269
 270         # so we hope to get the lock back soon!
 271
 272         if ($self->{shutdown_request}) {
 273
 274             my $service_count = $self->active_service_count();
 275
 276             if ($service_count > 0) {
 277                 $haenv->log('err', "get shutdown request in state 'lost_agent_lock' - " .
 278                             "detected $service_count running services");
 279
 280             } else {
 281
 282                 # all services are stopped, so we can close the watchdog
 283
 284                 if ($self->{ha_agent_wd}) {
 285                     $haenv->watchdog_close($self->{ha_agent_wd});
 286                     delete $self->{ha_agent_wd};
 287                 }
 288
 289                 return 0;
 290             }
 291         }
 292
 293         $haenv->sleep(5);
 294
 295     } else {
 296
 297         die "got unexpected status '$state'\n";
 298
 299     }
 300
 301     return 1;
 302 }
 303
 304 sub manage_resources {
 305     my ($self) = @_;
 306
 307     my $haenv = $self->{haenv};
 308
 309     my $nodename = $haenv->nodename();
 310
 311     my $ss = $self->{service_status};
 312
 313     foreach my $sid (keys %$ss) {
 314         my $sd = $ss->{$sid};
 315         next if !$sd->{node};
 316         next if !$sd->{uid};
 317         next if $sd->{node} ne $nodename;
 318         my $req_state = $sd->{state};
 319         next if !defined($req_state);
 320         next if $req_state eq 'freeze';
 321         eval {
 322             $self->queue_resource_command($sid, $sd->{uid}, $req_state, $sd->{target});
 323         };
 324         if (my $err = $@) {
 325             $haenv->log('err', "unable to run resource agent for '$sid' - $err"); # fixme
 326         }
 327     }
 328
 329     my $starttime = $haenv->get_time();
 330
 331     # start workers
 332     my $max_workers = 4;
 333
 334     my $sc = $haenv->read_service_config();
 335
 336     while (($haenv->get_time() - $starttime) < 5) {
 337         my $count =  $self->check_active_workers();
 338
 339         foreach my $sid (keys %{$self->{workers}}) {
 340             last if $count >= $max_workers;
 341             my $w = $self->{workers}->{$sid};
 342             my $cd = $sc->{$sid};
 343             if (!$cd) {
 344                 $haenv->log('err', "missing resource configuration for '$sid'");
 345                 next;
 346             }
 347             if (!$w->{pid}) {
 348                 if ($haenv->can_fork()) {
 349                     my $pid = fork();
 350                     if (!defined($pid)) {
 351                         $haenv->log('err', "fork worker failed");
 352                         $count = 0; last; # abort, try later
 353                     } elsif ($pid == 0) {
 354                         # do work
 355                         my $res = -1;
 356                         eval {
 357                             $res = $haenv->exec_resource_agent($sid, $cd, $w->{state}, $w->{target});
 358                         };
 359                         if (my $err = $@) {
 360                             $haenv->log('err', $err);
 361                             POSIX::_exit(-1);
 362                         }
 363                         POSIX::_exit($res);
 364                     } else {
 365                         $count++;
 366                         $w->{pid} = $pid;
 367                     }
 368                 } else {
 369                     my $res = -1;
 370                     eval {
 371                         $res = $haenv->exec_resource_agent($sid, $cd, $w->{state}, $w->{target});
 372                     };
 373                     if (my $err = $@) {
 374                         $haenv->log('err', $err);
 375                     }
 376                     $self->resource_command_finished($sid, $w->{uid}, $res);
 377                 }
 378             }
 379         }
 380
 381         last if !$count;
 382
 383         $haenv->sleep(1);
 384     }
 385 }
 386
 387 # fixme: use a queue an limit number of parallel workers?
 388 sub queue_resource_command {
 389     my ($self, $sid, $uid, $state, $target) = @_;
 390
 391     if (my $w = $self->{workers}->{$sid}) {
 392         return if $w->{pid}; # already started
 393         # else, delete and overwrite queue entry with new command
 394         delete $self->{workers}->{$sid};
 395     }
 396
 397     $self->{workers}->{$sid} = {
 398         sid => $sid,
 399         uid => $uid,
 400         state => $state,
 401     };
 402
 403     $self->{workers}->{$sid}->{target} = $target if $target;
 404 }
 405
 406 sub check_active_workers {
 407     my ($self) = @_;
 408
 409     # finish/count workers
 410     my $count = 0;
 411     foreach my $sid (keys %{$self->{workers}}) {
 412         my $w = $self->{workers}->{$sid};
 413         if (my $pid = $w->{pid}) {
 414             # check status
 415             my $waitpid = waitpid($pid, WNOHANG);
 416             if (defined($waitpid) && ($waitpid == $pid)) {
 417                 $self->resource_command_finished($sid, $w->{uid}, $?);
 418             } else {
 419                 $count++;
 420             }
 421         }
 422     }
 423
 424     return $count;
 425 }
 426
 427 sub resource_command_finished {
 428     my ($self, $sid, $uid, $status) = @_;
 429
 430     my $haenv = $self->{haenv};
 431
 432     my $w = delete $self->{workers}->{$sid};
 433     return if !$w; # should not happen
 434
 435     my $exit_code = -1;
 436
 437     if ($status == -1) {
 438         $haenv->log('err', "resource agent $sid finished - failed to execute");
 439     }  elsif (my $sig = ($status & 127)) {
 440         $haenv->log('err', "resource agent $sid finished - got signal $sig");
 441     } else {
 442         $exit_code = ($status >> 8);
 443     }
 444
 445     $self->{results}->{$uid} = {
 446         sid => $w->{sid},
 447         state => $w->{state},
 448         exit_code => $exit_code,
 449     };
 450
 451     my $ss = $self->{service_status};
 452
 453     # compute hash of valid/existing uids
 454     my $valid_uids = {};
 455     foreach my $sid (keys %$ss) {
 456         my $sd = $ss->{$sid};
 457         next if !$sd->{uid};
 458         $valid_uids->{$sd->{uid}} = 1;
 459     }
 460
 461     my $results = {};
 462     foreach my $id (keys %{$self->{results}}) {
 463         next if !$valid_uids->{$id};
 464         $results->{$id} = $self->{results}->{$id};
 465     }
 466     $self->{results} = $results;
 467
 468     $self->update_lrm_status();
 469 }
 470
 471 1;