src/PVE/HA/LRM.pm

   1 package PVE::HA::LRM;
   2
   3 # Local Resource Manager
   4
   5 use strict;
   6 use warnings;
   7 use Data::Dumper;
   8 use POSIX qw(:sys_wait_h);
   9
  10 use PVE::SafeSyslog;
  11 use PVE::Tools;
  12 use PVE::HA::Tools;
  13
  14 # Server can have several states:
  15
  16 my $valid_states = {
  17     wait_for_agent_lock => "waiting for agent lock",
  18     active => "got agent_lock",
  19     lost_agent_lock => "lost agent_lock",
  20 };
  21
  22 sub new {
  23     my ($this, $haenv) = @_;
  24
  25     my $class = ref($this) || $this;
  26
  27     my $self = bless {
  28         haenv => $haenv,
  29         status => { state => 'startup' },
  30         workers => {},
  31         results => {},
  32         shutdown_request => 0,
  33         # mode can be: active, reboot, shutdown, restart
  34         mode => 'active',
  35     }, $class;
  36
  37     $self->set_local_status({ state =>  'wait_for_agent_lock' });
  38
  39     return $self;
  40 }
  41
  42 sub shutdown_request {
  43     my ($self) = @_;
  44
  45     $self->{shutdown_request} = 1;
  46
  47     $self->{mode} = 'restart'; # fixme: detect shutdown/reboot
  48
  49     eval { $self->update_lrm_status(); };
  50     if (my $err = $@) {
  51         $self->log('err', "unable to update lrm status file - $err");
  52     }
  53 }
  54
  55 sub get_local_status {
  56     my ($self) = @_;
  57
  58     return $self->{status};
  59 }
  60
  61 sub set_local_status {
  62     my ($self, $new) = @_;
  63
  64     die "invalid state '$new->{state}'" if !$valid_states->{$new->{state}};
  65
  66     my $haenv = $self->{haenv};
  67
  68     my $old = $self->{status};
  69
  70     # important: only update if if really changed
  71     return if $old->{state} eq $new->{state};
  72
  73     $haenv->log('info', "status change $old->{state} => $new->{state}");
  74
  75     $new->{state_change_time} = $haenv->get_time();
  76
  77     $self->{status} = $new;
  78 }
  79
  80 sub update_lrm_status {
  81     my ($self) = @_;
  82
  83     my $haenv = $self->{haenv};
  84
  85     return 0 if !$haenv->quorate();
  86
  87     my $lrm_status = {
  88         mode => $self->{mode},
  89         results => $self->{results},
  90         timestamp => $haenv->get_time(),
  91     };
  92
  93     eval { $haenv->write_lrm_status($lrm_status); };
  94     if (my $err = $@) {
  95         $haenv->log('err', "unable to write lrm status file - $err");
  96         return 0;
  97     }
  98
  99     return 1;
 100 }
 101
 102 sub get_protected_ha_agent_lock {
 103     my ($self) = @_;
 104
 105     my $haenv = $self->{haenv};
 106
 107     my $count = 0;
 108     my $starttime = $haenv->get_time();
 109
 110     for (;;) {
 111
 112         if ($haenv->get_ha_agent_lock()) {
 113             if ($self->{ha_agent_wd}) {
 114                 $haenv->watchdog_update($self->{ha_agent_wd});
 115             } else {
 116                 my $wfh = $haenv->watchdog_open();
 117                 $self->{ha_agent_wd} = $wfh;
 118             }
 119             return 1;
 120         }
 121
 122         last if ++$count > 5; # try max 5 time
 123
 124         my $delay = $haenv->get_time() - $starttime;
 125         last if $delay > 5; # for max 5 seconds
 126
 127         $haenv->sleep(1);
 128     }
 129
 130     return 0;
 131 }
 132
 133 sub active_service_count {
 134     my ($self) = @_;
 135
 136     my $haenv = $self->{haenv};
 137
 138     my $nodename = $haenv->nodename();
 139
 140     my $ss = $self->{service_status};
 141
 142     my $count = 0;
 143
 144     foreach my $sid (keys %$ss) {
 145         my $sd = $ss->{$sid};
 146         next if !$sd->{node};
 147         next if $sd->{node} ne $nodename;
 148         my $req_state = $sd->{state};
 149         next if !defined($req_state);
 150         next if $req_state eq 'stopped';
 151         next if $req_state eq 'freeze';
 152
 153         $count++;
 154     }
 155
 156     return $count;
 157 }
 158
 159 my $wrote_lrm_status_at_startup = 0;
 160
 161 sub do_one_iteration {
 162     my ($self) = @_;
 163
 164     my $haenv = $self->{haenv};
 165
 166     if (!$wrote_lrm_status_at_startup) {
 167         if ($self->update_lrm_status()) {
 168             $wrote_lrm_status_at_startup = 1;
 169         } else {
 170             # do nothing
 171             $haenv->sleep(5);
 172             return $self->{shutdown_request} ? 0 : 1;
 173         }
 174     }
 175
 176     my $status = $self->get_local_status();
 177     my $state = $status->{state};
 178
 179     my $ms = $haenv->read_manager_status();
 180     $self->{service_status} =  $ms->{service_status} || {};
 181
 182     my $fence_request = PVE::HA::Tools::count_fenced_services($self->{service_status}, $haenv->nodename());
 183
 184     # do state changes first
 185
 186     my $ctime = $haenv->get_time();
 187
 188     if ($state eq 'wait_for_agent_lock') {
 189
 190         my $service_count = $self->active_service_count();
 191
 192         if (!$fence_request && $service_count && $haenv->quorate()) {
 193             if ($self->get_protected_ha_agent_lock()) {
 194                 $self->set_local_status({ state => 'active' });
 195             }
 196         }
 197
 198     } elsif ($state eq 'lost_agent_lock') {
 199
 200         if (!$fence_request && $haenv->quorate()) {
 201             if ($self->get_protected_ha_agent_lock()) {
 202                 $self->set_local_status({ state => 'active' });
 203             }
 204         }
 205
 206     } elsif ($state eq 'active') {
 207
 208         if ($fence_request) {
 209             $haenv->log('err', "node need to be fenced - releasing agent_lock\n");
 210             $self->set_local_status({ state => 'lost_agent_lock'});
 211         } elsif (!$self->get_protected_ha_agent_lock()) {
 212             $self->set_local_status({ state => 'lost_agent_lock'});
 213         }
 214     }
 215
 216     $status = $self->get_local_status();
 217     $state = $status->{state};
 218
 219     # do work
 220
 221     if ($state eq 'wait_for_agent_lock') {
 222
 223         return 0 if $self->{shutdown_request};
 224
 225         $self->update_lrm_status();
 226
 227         $haenv->sleep(5);
 228
 229     } elsif ($state eq 'active') {
 230
 231         my $startime = $haenv->get_time();
 232
 233         my $max_time = 10;
 234
 235         my $shutdown = 0;
 236
 237         # do work (max_time seconds)
 238         eval {
 239             # fixme: set alert timer
 240
 241             if ($self->{shutdown_request}) {
 242
 243                 # fixme: request service stop or relocate ?
 244
 245                 my $service_count = $self->active_service_count();
 246
 247                 if ($service_count == 0) {
 248
 249                     if ($self->{ha_agent_wd}) {
 250                         $haenv->watchdog_close($self->{ha_agent_wd});
 251                         delete $self->{ha_agent_wd};
 252                     }
 253
 254                     $shutdown = 1;
 255                 }
 256             } else {
 257
 258                 $self->manage_resources();
 259
 260             }
 261         };
 262         if (my $err = $@) {
 263             $haenv->log('err', "got unexpected error - $err");
 264         }
 265
 266         $self->update_lrm_status();
 267
 268         return 0 if $shutdown;
 269
 270         $haenv->sleep_until($startime + $max_time);
 271
 272     } elsif ($state eq 'lost_agent_lock') {
 273
 274         # Note: watchdog is active an will triger soon!
 275
 276         # so we hope to get the lock back soon!
 277
 278         if ($self->{shutdown_request}) {
 279
 280             my $service_count = $self->active_service_count();
 281
 282             if ($service_count > 0) {
 283                 $haenv->log('err', "get shutdown request in state 'lost_agent_lock' - " .
 284                             "detected $service_count running services");
 285
 286             } else {
 287
 288                 # all services are stopped, so we can close the watchdog
 289
 290                 if ($self->{ha_agent_wd}) {
 291                     $haenv->watchdog_close($self->{ha_agent_wd});
 292                     delete $self->{ha_agent_wd};
 293                 }
 294
 295                 return 0;
 296             }
 297         }
 298
 299         $haenv->sleep(5);
 300
 301     } else {
 302
 303         die "got unexpected status '$state'\n";
 304
 305     }
 306
 307     return 1;
 308 }
 309
 310 sub manage_resources {
 311     my ($self) = @_;
 312
 313     my $haenv = $self->{haenv};
 314
 315     my $nodename = $haenv->nodename();
 316
 317     my $ss = $self->{service_status};
 318
 319     foreach my $sid (keys %$ss) {
 320         my $sd = $ss->{$sid};
 321         next if !$sd->{node};
 322         next if !$sd->{uid};
 323         next if $sd->{node} ne $nodename;
 324         my $req_state = $sd->{state};
 325         next if !defined($req_state);
 326         next if $req_state eq 'freeze';
 327         eval {
 328             $self->queue_resource_command($sid, $sd->{uid}, $req_state, $sd->{target});
 329         };
 330         if (my $err = $@) {
 331             $haenv->log('err', "unable to run resource agent for '$sid' - $err"); # fixme
 332         }
 333     }
 334
 335     my $starttime = $haenv->get_time();
 336
 337     # start workers
 338     my $max_workers = 4;
 339
 340     my $sc = $haenv->read_service_config();
 341
 342     while (($haenv->get_time() - $starttime) < 5) {
 343         my $count =  $self->check_active_workers();
 344
 345         foreach my $sid (keys %{$self->{workers}}) {
 346             last if $count >= $max_workers;
 347             my $w = $self->{workers}->{$sid};
 348             my $cd = $sc->{$sid};
 349             if (!$cd) {
 350                 $haenv->log('err', "missing resource configuration for '$sid'");
 351                 next;
 352             }
 353             if (!$w->{pid}) {
 354                 if ($haenv->can_fork()) {
 355                     my $pid = fork();
 356                     if (!defined($pid)) {
 357                         $haenv->log('err', "fork worker failed");
 358                         $count = 0; last; # abort, try later
 359                     } elsif ($pid == 0) {
 360                         # do work
 361                         my $res = -1;
 362                         eval {
 363                             $res = $haenv->exec_resource_agent($sid, $cd, $w->{state}, $w->{target});
 364                         };
 365                         if (my $err = $@) {
 366                             $haenv->log('err', $err);
 367                             POSIX::_exit(-1);
 368                         }
 369                         POSIX::_exit($res);
 370                     } else {
 371                         $count++;
 372                         $w->{pid} = $pid;
 373                     }
 374                 } else {
 375                     my $res = -1;
 376                     eval {
 377                         $res = $haenv->exec_resource_agent($sid, $cd, $w->{state}, $w->{target});
 378                     };
 379                     if (my $err = $@) {
 380                         $haenv->log('err', $err);
 381                     }
 382                     $self->resource_command_finished($sid, $w->{uid}, $res);
 383                 }
 384             }
 385         }
 386
 387         last if !$count;
 388
 389         $haenv->sleep(1);
 390     }
 391 }
 392
 393 # fixme: use a queue an limit number of parallel workers?
 394 sub queue_resource_command {
 395     my ($self, $sid, $uid, $state, $target) = @_;
 396
 397     if (my $w = $self->{workers}->{$sid}) {
 398         return if $w->{pid}; # already started
 399         # else, delete and overwrite queue entry with new command
 400         delete $self->{workers}->{$sid};
 401     }
 402
 403     $self->{workers}->{$sid} = {
 404         sid => $sid,
 405         uid => $uid,
 406         state => $state,
 407     };
 408
 409     $self->{workers}->{$sid}->{target} = $target if $target;
 410 }
 411
 412 sub check_active_workers {
 413     my ($self) = @_;
 414
 415     # finish/count workers
 416     my $count = 0;
 417     foreach my $sid (keys %{$self->{workers}}) {
 418         my $w = $self->{workers}->{$sid};
 419         if (my $pid = $w->{pid}) {
 420             # check status
 421             my $waitpid = waitpid($pid, WNOHANG);
 422             if (defined($waitpid) && ($waitpid == $pid)) {
 423                 $self->resource_command_finished($sid, $w->{uid}, $?);
 424             } else {
 425                 $count++;
 426             }
 427         }
 428     }
 429
 430     return $count;
 431 }
 432
 433 sub resource_command_finished {
 434     my ($self, $sid, $uid, $status) = @_;
 435
 436     my $haenv = $self->{haenv};
 437
 438     my $w = delete $self->{workers}->{$sid};
 439     return if !$w; # should not happen
 440
 441     my $exit_code = -1;
 442
 443     if ($status == -1) {
 444         $haenv->log('err', "resource agent $sid finished - failed to execute");
 445     }  elsif (my $sig = ($status & 127)) {
 446         $haenv->log('err', "resource agent $sid finished - got signal $sig");
 447     } else {
 448         $exit_code = ($status >> 8);
 449     }
 450
 451     $self->{results}->{$uid} = {
 452         sid => $w->{sid},
 453         state => $w->{state},
 454         exit_code => $exit_code,
 455     };
 456
 457     my $ss = $self->{service_status};
 458
 459     # compute hash of valid/existing uids
 460     my $valid_uids = {};
 461     foreach my $sid (keys %$ss) {
 462         my $sd = $ss->{$sid};
 463         next if !$sd->{uid};
 464         $valid_uids->{$sd->{uid}} = 1;
 465     }
 466
 467     my $results = {};
 468     foreach my $id (keys %{$self->{results}}) {
 469         next if !$valid_uids->{$id};
 470         $results->{$id} = $self->{results}->{$id};
 471     }
 472     $self->{results} = $results;
 473 }
 474
 475 1;