src/PVE/HA/LRM.pm

   1 package PVE::HA::LRM;
   2
   3 # Local Resource Manager
   4
   5 use strict;
   6 use warnings;
   7 use POSIX qw(:sys_wait_h);
   8
   9 use PVE::SafeSyslog;
  10 use PVE::Tools;
  11 use PVE::HA::Tools ':exit_codes';
  12 use PVE::HA::Resources;
  13
  14 # Server can have several states:
  15
  16 my $valid_states = {
  17     wait_for_agent_lock => "waiting for agent lock",
  18     active => "got agent_lock",
  19     lost_agent_lock => "lost agent_lock",
  20 };
  21
  22 sub new {
  23     my ($this, $haenv) = @_;
  24
  25     my $class = ref($this) || $this;
  26
  27     my $self = bless {
  28         haenv => $haenv,
  29         status => { state => 'startup' },
  30         workers => {},
  31         results => {},
  32         restart_tries => {},
  33         shutdown_request => 0,
  34         shutdown_errors => 0,
  35         # mode can be: active, reboot, shutdown, restart
  36         mode => 'active',
  37     }, $class;
  38
  39     $self->set_local_status({ state =>  'wait_for_agent_lock' });
  40
  41     return $self;
  42 }
  43
  44 sub shutdown_request {
  45     my ($self) = @_;
  46
  47     return if $self->{shutdown_request}; # already in shutdown mode
  48
  49     my $haenv = $self->{haenv};
  50
  51     my $nodename = $haenv->nodename();
  52
  53     my ($shutdown, $reboot) = $haenv->is_node_shutdown();
  54
  55     if ($shutdown) {
  56         # *always* queue stop jobs for all services if the node shuts down,
  57         # independent if it's a reboot or a poweroff, else we may corrupt
  58         # services or hinder node shutdown
  59         my $ss = $self->{service_status};
  60
  61         foreach my $sid (keys %$ss) {
  62             my $sd = $ss->{$sid};
  63             next if !$sd->{node};
  64             next if $sd->{node} ne $nodename;
  65             # Note: use undef uid to mark shutdown/stop jobs
  66             $self->queue_resource_command($sid, undef, 'request_stop');
  67         }
  68     }
  69
  70     if ($shutdown) {
  71         if ($reboot) {
  72             $haenv->log('info', "reboot LRM, stop and freeze all services");
  73             $self->{mode} = 'restart';
  74         } else {
  75             $haenv->log('info', "shutdown LRM, stop all services");
  76             $self->{mode} = 'shutdown';
  77         }
  78     } else {
  79         $haenv->log('info', "restart LRM, freeze all services");
  80         $self->{mode} = 'restart';
  81     }
  82
  83     $self->{shutdown_request} = 1;
  84
  85     eval { $self->update_lrm_status(); };
  86     if (my $err = $@) {
  87         $self->log('err', "unable to update lrm status file - $err");
  88     }
  89 }
  90
  91 sub get_local_status {
  92     my ($self) = @_;
  93
  94     return $self->{status};
  95 }
  96
  97 sub set_local_status {
  98     my ($self, $new) = @_;
  99
 100     die "invalid state '$new->{state}'" if !$valid_states->{$new->{state}};
 101
 102     my $haenv = $self->{haenv};
 103
 104     my $old = $self->{status};
 105
 106     # important: only update if if really changed
 107     return if $old->{state} eq $new->{state};
 108
 109     $haenv->log('info', "status change $old->{state} => $new->{state}");
 110
 111     $new->{state_change_time} = $haenv->get_time();
 112
 113     $self->{status} = $new;
 114 }
 115
 116 sub update_lrm_status {
 117     my ($self) = @_;
 118
 119     my $haenv = $self->{haenv};
 120
 121     return 0 if !$haenv->quorate();
 122
 123     my $lrm_status = {
 124         state => $self->{status}->{state},
 125         mode => $self->{mode},
 126         results => $self->{results},
 127         timestamp => $haenv->get_time(),
 128     };
 129
 130     eval { $haenv->write_lrm_status($lrm_status); };
 131     if (my $err = $@) {
 132         $haenv->log('err', "unable to write lrm status file - $err");
 133         return 0;
 134     }
 135
 136     return 1;
 137 }
 138
 139 sub update_service_status {
 140     my ($self) = @_;
 141
 142     my $haenv = $self->{haenv};
 143
 144     my $ms = eval { $haenv->read_manager_status(); };
 145     if (my $err = $@) {
 146         $haenv->log('err', "updating service status from manager failed: $err");
 147         return undef;
 148     } else {
 149         $self->{service_status} = $ms->{service_status} || {};
 150         return 1;
 151     }
 152 }
 153
 154 sub get_protected_ha_agent_lock {
 155     my ($self) = @_;
 156
 157     my $haenv = $self->{haenv};
 158
 159     my $count = 0;
 160     my $starttime = $haenv->get_time();
 161
 162     for (;;) {
 163
 164         if ($haenv->get_ha_agent_lock()) {
 165             if ($self->{ha_agent_wd}) {
 166                 $haenv->watchdog_update($self->{ha_agent_wd});
 167             } else {
 168                 my $wfh = $haenv->watchdog_open();
 169                 $self->{ha_agent_wd} = $wfh;
 170             }
 171             return 1;
 172         }
 173
 174         last if ++$count > 5; # try max 5 time
 175
 176         my $delay = $haenv->get_time() - $starttime;
 177         last if $delay > 5; # for max 5 seconds
 178
 179         $haenv->sleep(1);
 180     }
 181
 182     return 0;
 183 }
 184
 185 sub active_service_count {
 186     my ($self) = @_;
 187
 188     my $haenv = $self->{haenv};
 189
 190     my $nodename = $haenv->nodename();
 191
 192     my $ss = $self->{service_status};
 193
 194     my $count = 0;
 195
 196     foreach my $sid (keys %$ss) {
 197         my $sd = $ss->{$sid};
 198         next if !$sd->{node};
 199         next if $sd->{node} ne $nodename;
 200         my $req_state = $sd->{state};
 201         next if !defined($req_state);
 202         next if $req_state eq 'stopped';
 203         next if $req_state eq 'freeze';
 204         # erroneous services are not managed by HA, don't count them as active
 205         next if $req_state eq 'error';
 206
 207         $count++;
 208     }
 209
 210     return $count;
 211 }
 212
 213 my $wrote_lrm_status_at_startup = 0;
 214
 215 sub do_one_iteration {
 216     my ($self) = @_;
 217
 218     my $haenv = $self->{haenv};
 219
 220     $haenv->loop_start_hook();
 221
 222     my $res = $self->work();
 223
 224     $haenv->loop_end_hook();
 225
 226     return $res;
 227 }
 228
 229 sub work {
 230     my ($self) = @_;
 231
 232     my $haenv = $self->{haenv};
 233
 234     if (!$wrote_lrm_status_at_startup) {
 235         if ($self->update_lrm_status()) {
 236             $wrote_lrm_status_at_startup = 1;
 237         } else {
 238             # do nothing
 239             $haenv->sleep(5);
 240             return $self->{shutdown_request} ? 0 : 1;
 241         }
 242     }
 243
 244     my $status = $self->get_local_status();
 245     my $state = $status->{state};
 246
 247     $self->update_service_status();
 248
 249     my $fence_request = PVE::HA::Tools::count_fenced_services($self->{service_status}, $haenv->nodename());
 250
 251     # do state changes first
 252
 253     my $ctime = $haenv->get_time();
 254
 255     if ($state eq 'wait_for_agent_lock') {
 256
 257         my $service_count = $self->active_service_count();
 258
 259         if (!$fence_request && $service_count && $haenv->quorate()) {
 260             if ($self->get_protected_ha_agent_lock()) {
 261                 $self->set_local_status({ state => 'active' });
 262             }
 263         }
 264
 265     } elsif ($state eq 'lost_agent_lock') {
 266
 267         if (!$fence_request && $haenv->quorate()) {
 268             if ($self->get_protected_ha_agent_lock()) {
 269                 $self->set_local_status({ state => 'active' });
 270             }
 271         }
 272
 273     } elsif ($state eq 'active') {
 274
 275         if ($fence_request) {
 276             $haenv->log('err', "node need to be fenced - releasing agent_lock\n");
 277             $self->set_local_status({ state => 'lost_agent_lock'});
 278         } elsif (!$self->get_protected_ha_agent_lock()) {
 279             $self->set_local_status({ state => 'lost_agent_lock'});
 280         }
 281     }
 282
 283     $status = $self->get_local_status();
 284     $state = $status->{state};
 285
 286     # do work
 287
 288     if ($state eq 'wait_for_agent_lock') {
 289
 290         return 0 if $self->{shutdown_request};
 291
 292         $self->update_lrm_status();
 293
 294         $haenv->sleep(5);
 295
 296     } elsif ($state eq 'active') {
 297
 298         my $startime = $haenv->get_time();
 299
 300         my $max_time = 10;
 301
 302         my $shutdown = 0;
 303
 304         # do work (max_time seconds)
 305         eval {
 306             # fixme: set alert timer
 307
 308             # if we could not get the current service status there's no point
 309             # in doing anything, try again next round.
 310             return if !$self->update_service_status();
 311
 312             if ($self->{shutdown_request}) {
 313
 314                 if ($self->{mode} eq 'restart') {
 315
 316                     my $service_count = $self->active_service_count();
 317
 318                     if ($service_count == 0) {
 319
 320                         if ($self->run_workers() == 0) {
 321                             if ($self->{ha_agent_wd}) {
 322                                 $haenv->watchdog_close($self->{ha_agent_wd});
 323                                 delete $self->{ha_agent_wd};
 324                             }
 325
 326                             $shutdown = 1;
 327
 328                             # restart with no or freezed services, release the lock
 329                             $haenv->release_ha_agent_lock();
 330                         }
 331                     }
 332                 } else {
 333
 334                     if ($self->run_workers() == 0) {
 335                         if ($self->{shutdown_errors} == 0) {
 336                             if ($self->{ha_agent_wd}) {
 337                                 $haenv->watchdog_close($self->{ha_agent_wd});
 338                                 delete $self->{ha_agent_wd};
 339                             }
 340
 341                             # shutdown with all services stopped thus release the lock
 342                             $haenv->release_ha_agent_lock();
 343                         }
 344
 345                         $shutdown = 1;
 346                     }
 347                 }
 348             } else {
 349
 350                 $self->manage_resources();
 351
 352             }
 353         };
 354         if (my $err = $@) {
 355             $haenv->log('err', "got unexpected error - $err");
 356         }
 357
 358         $self->update_lrm_status();
 359
 360         return 0 if $shutdown;
 361
 362         $haenv->sleep_until($startime + $max_time);
 363
 364     } elsif ($state eq 'lost_agent_lock') {
 365
 366         # Note: watchdog is active an will triger soon!
 367
 368         # so we hope to get the lock back soon!
 369
 370         if ($self->{shutdown_request}) {
 371
 372             my $service_count = $self->active_service_count();
 373
 374             if ($service_count > 0) {
 375                 $haenv->log('err', "get shutdown request in state 'lost_agent_lock' - " .
 376                             "detected $service_count running services");
 377
 378             } else {
 379
 380                 # all services are stopped, so we can close the watchdog
 381
 382                 if ($self->{ha_agent_wd}) {
 383                     $haenv->watchdog_close($self->{ha_agent_wd});
 384                     delete $self->{ha_agent_wd};
 385                 }
 386
 387                 return 0;
 388             }
 389         }
 390
 391         $haenv->sleep(5);
 392
 393     } else {
 394
 395         die "got unexpected status '$state'\n";
 396
 397     }
 398
 399     return 1;
 400 }
 401
 402 sub run_workers {
 403     my ($self) = @_;
 404
 405     my $haenv = $self->{haenv};
 406
 407     my $starttime = $haenv->get_time();
 408
 409     # number of workers to start, if 0 we exec the command directly witouth forking
 410     my $max_workers = $haenv->get_max_workers();
 411
 412     my $sc = $haenv->read_service_config();
 413
 414     while (($haenv->get_time() - $starttime) < 5) {
 415         my $count =  $self->check_active_workers();
 416
 417         foreach my $sid (sort keys %{$self->{workers}}) {
 418             last if $count >= $max_workers && $max_workers > 0;
 419
 420             my $w = $self->{workers}->{$sid};
 421             if (!$w->{pid}) {
 422                 # only fork if we may else call exec_resource_agent
 423                 # directly (e.g. for regression tests)
 424                 if ($max_workers > 0) {
 425                     my $pid = fork();
 426                     if (!defined($pid)) {
 427                         $haenv->log('err', "fork worker failed");
 428                         $count = 0; last; # abort, try later
 429                     } elsif ($pid == 0) {
 430                         $haenv->after_fork(); # cleanup
 431
 432                         # do work
 433                         my $res = -1;
 434                         eval {
 435                             $res = $self->exec_resource_agent($sid, $sc->{$sid}, $w->{state}, $w->{target});
 436                         };
 437                         if (my $err = $@) {
 438                             $haenv->log('err', $err);
 439                             POSIX::_exit(-1);
 440                         }
 441                         POSIX::_exit($res);
 442                     } else {
 443                         $count++;
 444                         $w->{pid} = $pid;
 445                     }
 446                 } else {
 447                     my $res = -1;
 448                     eval {
 449                         $res = $self->exec_resource_agent($sid, $sc->{$sid}, $w->{state}, $w->{target});
 450                         $res = $res << 8 if $res > 0;
 451                     };
 452                     if (my $err = $@) {
 453                         $haenv->log('err', $err);
 454                     }
 455                     if (defined($w->{uid})) {
 456                         $self->resource_command_finished($sid, $w->{uid}, $res);
 457                     } else {
 458                         $self->stop_command_finished($sid, $res);
 459                     }
 460                 }
 461             }
 462         }
 463
 464         last if !$count;
 465
 466         $haenv->sleep(1);
 467     }
 468
 469     return scalar(keys %{$self->{workers}});
 470 }
 471
 472 sub manage_resources {
 473     my ($self) = @_;
 474
 475     my $haenv = $self->{haenv};
 476
 477     my $nodename = $haenv->nodename();
 478
 479     my $ss = $self->{service_status};
 480
 481     foreach my $sid (keys %{$self->{restart_tries}}) {
 482         delete $self->{restart_tries}->{$sid} if !$ss->{$sid};
 483     }
 484
 485     foreach my $sid (keys %$ss) {
 486         my $sd = $ss->{$sid};
 487         next if !$sd->{node};
 488         next if !$sd->{uid};
 489         next if $sd->{node} ne $nodename;
 490         my $req_state = $sd->{state};
 491         next if !defined($req_state);
 492         next if $req_state eq 'freeze';
 493         $self->queue_resource_command($sid, $sd->{uid}, $req_state, $sd->{target});
 494     }
 495
 496     return $self->run_workers();
 497 }
 498
 499 sub queue_resource_command {
 500     my ($self, $sid, $uid, $state, $target) = @_;
 501
 502     # do not queue the excatly same command twice as this may lead to
 503     # an inconsistent HA state when the first command fails but the CRM
 504     # does not process its failure right away and the LRM starts a second
 505     # try, without the CRM knowing of it (race condition)
 506     # The 'stopped' command is an exception as we do not process its result
 507     # in the CRM and we want to execute it always (even with no active CRM)
 508     return if $state ne 'stopped' && $uid && defined($self->{results}->{$uid});
 509
 510     if (my $w = $self->{workers}->{$sid}) {
 511         return if $w->{pid}; # already started
 512         # else, delete and overwrite queue entry with new command
 513         delete $self->{workers}->{$sid};
 514     }
 515
 516     $self->{workers}->{$sid} = {
 517         sid => $sid,
 518         uid => $uid,
 519         state => $state,
 520     };
 521
 522     $self->{workers}->{$sid}->{target} = $target if $target;
 523 }
 524
 525 sub check_active_workers {
 526     my ($self) = @_;
 527
 528     # finish/count workers
 529     my $count = 0;
 530     foreach my $sid (keys %{$self->{workers}}) {
 531         my $w = $self->{workers}->{$sid};
 532         if (my $pid = $w->{pid}) {
 533             # check status
 534             my $waitpid = waitpid($pid, WNOHANG);
 535             if (defined($waitpid) && ($waitpid == $pid)) {
 536                 if (defined($w->{uid})) {
 537                     $self->resource_command_finished($sid, $w->{uid}, $?);
 538                 } else {
 539                     $self->stop_command_finished($sid, $?);
 540                 }
 541             } else {
 542                 $count++;
 543             }
 544         }
 545     }
 546
 547     return $count;
 548 }
 549
 550 sub stop_command_finished {
 551     my ($self, $sid, $status) = @_;
 552
 553     my $haenv = $self->{haenv};
 554
 555     my $w = delete $self->{workers}->{$sid};
 556     return if !$w; # should not happen
 557
 558     my $exit_code = -1;
 559
 560     if ($status == -1) {
 561         $haenv->log('err', "resource agent $sid finished - failed to execute");
 562     }  elsif (my $sig = ($status & 127)) {
 563         $haenv->log('err', "resource agent $sid finished - got signal $sig");
 564     } else {
 565         $exit_code = ($status >> 8);
 566     }
 567
 568     if ($exit_code != 0) {
 569         $self->{shutdown_errors}++;
 570     }
 571 }
 572
 573 sub resource_command_finished {
 574     my ($self, $sid, $uid, $status) = @_;
 575
 576     my $haenv = $self->{haenv};
 577
 578     my $w = delete $self->{workers}->{$sid};
 579     return if !$w; # should not happen
 580
 581     my $exit_code = -1;
 582
 583     if ($status == -1) {
 584         $haenv->log('err', "resource agent $sid finished - failed to execute");
 585     }  elsif (my $sig = ($status & 127)) {
 586         $haenv->log('err', "resource agent $sid finished - got signal $sig");
 587     } else {
 588         $exit_code = ($status >> 8);
 589     }
 590
 591     $exit_code = $self->handle_service_exitcode($sid, $w->{state}, $exit_code);
 592
 593     return if $exit_code == ETRY_AGAIN; # tell nobody, simply retry
 594
 595     $self->{results}->{$uid} = {
 596         sid => $w->{sid},
 597         state => $w->{state},
 598         exit_code => $exit_code,
 599     };
 600
 601     my $ss = $self->{service_status};
 602
 603     # compute hash of valid/existing uids
 604     my $valid_uids = {};
 605     foreach my $sid (keys %$ss) {
 606         my $sd = $ss->{$sid};
 607         next if !$sd->{uid};
 608         $valid_uids->{$sd->{uid}} = 1;
 609     }
 610
 611     my $results = {};
 612     foreach my $id (keys %{$self->{results}}) {
 613         next if !$valid_uids->{$id};
 614         $results->{$id} = $self->{results}->{$id};
 615     }
 616     $self->{results} = $results;
 617 }
 618
 619 # processes the exit code from a finished resource agent, so that the CRM knows
 620 # if the LRM wants to retry an action based on the current recovery policies for
 621 # the failed service, or the CRM itself must try to recover from the failure.
 622 sub handle_service_exitcode {
 623     my ($self, $sid, $cmd, $exit_code) = @_;
 624
 625     my $haenv = $self->{haenv};
 626     my $tries = $self->{restart_tries};
 627
 628     my $sc = $haenv->read_service_config();
 629
 630     my $max_restart = 0;
 631
 632     if (my $cd = $sc->{$sid}) {
 633         $max_restart = $cd->{max_restart};
 634     }
 635
 636     if ($cmd eq 'started') {
 637
 638         if ($exit_code == SUCCESS) {
 639
 640             $tries->{$sid} = 0;
 641
 642             return $exit_code;
 643
 644         } elsif ($exit_code == ERROR) {
 645
 646             $tries->{$sid} = 0 if !defined($tries->{$sid});
 647
 648             if ($tries->{$sid} >= $max_restart) {
 649                 $haenv->log('err', "unable to start service $sid on local node".
 650                            " after $tries->{$sid} retries");
 651                 $tries->{$sid} = 0;
 652                 return ERROR;
 653             }
 654
 655             $tries->{$sid}++;
 656
 657             $haenv->log('warning', "restart policy: retry number $tries->{$sid}" .
 658                         " for service '$sid'");
 659             # tell CRM that we retry the start
 660             return ETRY_AGAIN;
 661         }
 662     }
 663
 664     return $exit_code;
 665
 666 }
 667
 668 sub exec_resource_agent {
 669     my ($self, $sid, $service_config, $cmd, @params) = @_;
 670
 671     # setup execution environment
 672
 673     $ENV{'PATH'} = '/sbin:/bin:/usr/sbin:/usr/bin';
 674
 675     my $haenv = $self->{haenv};
 676
 677     my $nodename = $haenv->nodename();
 678
 679     my (undef, $service_type, $service_name) = PVE::HA::Tools::parse_sid($sid);
 680
 681     my $plugin = PVE::HA::Resources->lookup($service_type);
 682     if (!$plugin) {
 683         $haenv->log('err', "service type '$service_type' not implemented");
 684         return EUNKNOWN_SERVICE_TYPE;
 685     }
 686
 687     if (!$service_config) {
 688         $haenv->log('err', "missing resource configuration for '$sid'");
 689         return EUNKNOWN_SERVICE;
 690     }
 691
 692     # process error state early
 693     if ($cmd eq 'error') {
 694
 695         $haenv->log('err', "service $sid is in an error state and needs manual " .
 696                     "intervention. Look up 'ERROR RECOVERY' in the documentation.");
 697
 698         return SUCCESS; # error always succeeds
 699     }
 700
 701     if ($service_config->{node} ne $nodename) {
 702         $haenv->log('err', "service '$sid' not on this node");
 703         return EWRONG_NODE;
 704     }
 705
 706     my $id = $service_name;
 707
 708     my $running = $plugin->check_running($haenv, $id);
 709
 710     if ($cmd eq 'started') {
 711
 712         return SUCCESS if $running;
 713
 714         $haenv->log("info", "starting service $sid");
 715
 716         $plugin->start($haenv, $id);
 717
 718         $running = $plugin->check_running($haenv, $id);
 719
 720         if ($running) {
 721             $haenv->log("info", "service status $sid started");
 722             return SUCCESS;
 723         } else {
 724             $haenv->log("warning", "unable to start service $sid");
 725             return ERROR;
 726         }
 727
 728     } elsif ($cmd eq 'request_stop' || $cmd eq 'stopped') {
 729
 730         return SUCCESS if !$running;
 731
 732         $haenv->log("info", "stopping service $sid");
 733
 734         $plugin->shutdown($haenv, $id);
 735
 736         $running = $plugin->check_running($haenv, $id);
 737
 738         if (!$running) {
 739             $haenv->log("info", "service status $sid stopped");
 740             return SUCCESS;
 741         } else {
 742             $haenv->log("info", "unable to stop stop service $sid (still running)");
 743             return ERROR;
 744         }
 745
 746     } elsif ($cmd eq 'migrate' || $cmd eq 'relocate') {
 747
 748         my $target = $params[0];
 749         if (!defined($target)) {
 750             die "$cmd '$sid' failed - missing target\n" if !defined($target);
 751             return EINVALID_PARAMETER;
 752         }
 753
 754         if ($service_config->{node} eq $target) {
 755             # already there
 756             return SUCCESS;
 757         }
 758
 759         my $online = ($cmd eq 'migrate') ? 1 : 0;
 760
 761         my $res = $plugin->migrate($haenv, $id, $target, $online);
 762
 763         # something went wrong if service is still on this node
 764         if (!$res) {
 765             $haenv->log("err", "service $sid not moved (migration error)");
 766             return ERROR;
 767         }
 768
 769         return SUCCESS;
 770
 771     }
 772
 773     $haenv->log("err", "implement me (cmd '$cmd')");
 774     return EUNKNOWN_COMMAND;
 775 }
 776
 777
 778 1;