src/PVE/HA/LRM.pm

   1 package PVE::HA::LRM;
   2
   3 # Local Resource Manager
   4
   5 use strict;
   6 use warnings;
   7 use Data::Dumper;
   8 use POSIX qw(:sys_wait_h);
   9
  10 use PVE::SafeSyslog;
  11 use PVE::Tools;
  12 use PVE::HA::Tools ':exit_codes';
  13 use PVE::HA::Resources;
  14
  15 # Server can have several states:
  16
  17 my $valid_states = {
  18     wait_for_agent_lock => "waiting for agent lock",
  19     active => "got agent_lock",
  20     lost_agent_lock => "lost agent_lock",
  21 };
  22
  23 sub new {
  24     my ($this, $haenv) = @_;
  25
  26     my $class = ref($this) || $this;
  27
  28     my $self = bless {
  29         haenv => $haenv,
  30         status => { state => 'startup' },
  31         workers => {},
  32         results => {},
  33         restart_tries => {},
  34         shutdown_request => 0,
  35         shutdown_errors => 0,
  36         # mode can be: active, reboot, shutdown, restart
  37         mode => 'active',
  38     }, $class;
  39
  40     $self->set_local_status({ state =>  'wait_for_agent_lock' });
  41
  42     return $self;
  43 }
  44
  45 sub shutdown_request {
  46     my ($self) = @_;
  47
  48     return if $self->{shutdown_request}; # already in shutdown mode
  49
  50     my $haenv = $self->{haenv};
  51
  52     my $nodename = $haenv->nodename();
  53
  54     my $shutdown = $haenv->is_node_shutdown();
  55
  56     if ($shutdown) {
  57         $haenv->log('info', "shutdown LRM, stop all services");
  58         $self->{mode} = 'shutdown';
  59
  60         # queue stop jobs for all services
  61
  62         my $ss = $self->{service_status};
  63
  64         foreach my $sid (keys %$ss) {
  65             my $sd = $ss->{$sid};
  66             next if !$sd->{node};
  67             next if $sd->{node} ne $nodename;
  68             # Note: use undef uid to mark shutdown/stop jobs
  69             $self->queue_resource_command($sid, undef, 'request_stop');
  70         }
  71
  72     } else {
  73         $haenv->log('info', "restart LRM, freeze all services");
  74         $self->{mode} = 'restart';
  75     }
  76
  77     $self->{shutdown_request} = 1;
  78
  79     eval { $self->update_lrm_status(); };
  80     if (my $err = $@) {
  81         $self->log('err', "unable to update lrm status file - $err");
  82     }
  83 }
  84
  85 sub get_local_status {
  86     my ($self) = @_;
  87
  88     return $self->{status};
  89 }
  90
  91 sub set_local_status {
  92     my ($self, $new) = @_;
  93
  94     die "invalid state '$new->{state}'" if !$valid_states->{$new->{state}};
  95
  96     my $haenv = $self->{haenv};
  97
  98     my $old = $self->{status};
  99
 100     # important: only update if if really changed
 101     return if $old->{state} eq $new->{state};
 102
 103     $haenv->log('info', "status change $old->{state} => $new->{state}");
 104
 105     $new->{state_change_time} = $haenv->get_time();
 106
 107     $self->{status} = $new;
 108 }
 109
 110 sub update_lrm_status {
 111     my ($self) = @_;
 112
 113     my $haenv = $self->{haenv};
 114
 115     return 0 if !$haenv->quorate();
 116
 117     my $lrm_status = {
 118         mode => $self->{mode},
 119         results => $self->{results},
 120         timestamp => $haenv->get_time(),
 121     };
 122
 123     eval { $haenv->write_lrm_status($lrm_status); };
 124     if (my $err = $@) {
 125         $haenv->log('err', "unable to write lrm status file - $err");
 126         return 0;
 127     }
 128
 129     return 1;
 130 }
 131
 132 sub get_protected_ha_agent_lock {
 133     my ($self) = @_;
 134
 135     my $haenv = $self->{haenv};
 136
 137     my $count = 0;
 138     my $starttime = $haenv->get_time();
 139
 140     for (;;) {
 141
 142         if ($haenv->get_ha_agent_lock()) {
 143             if ($self->{ha_agent_wd}) {
 144                 $haenv->watchdog_update($self->{ha_agent_wd});
 145             } else {
 146                 my $wfh = $haenv->watchdog_open();
 147                 $self->{ha_agent_wd} = $wfh;
 148             }
 149             return 1;
 150         }
 151
 152         last if ++$count > 5; # try max 5 time
 153
 154         my $delay = $haenv->get_time() - $starttime;
 155         last if $delay > 5; # for max 5 seconds
 156
 157         $haenv->sleep(1);
 158     }
 159
 160     return 0;
 161 }
 162
 163 sub active_service_count {
 164     my ($self) = @_;
 165
 166     my $haenv = $self->{haenv};
 167
 168     my $nodename = $haenv->nodename();
 169
 170     my $ss = $self->{service_status};
 171
 172     my $count = 0;
 173
 174     foreach my $sid (keys %$ss) {
 175         my $sd = $ss->{$sid};
 176         next if !$sd->{node};
 177         next if $sd->{node} ne $nodename;
 178         my $req_state = $sd->{state};
 179         next if !defined($req_state);
 180         next if $req_state eq 'stopped';
 181         next if $req_state eq 'freeze';
 182
 183         $count++;
 184     }
 185
 186     return $count;
 187 }
 188
 189 my $wrote_lrm_status_at_startup = 0;
 190
 191 sub do_one_iteration {
 192     my ($self) = @_;
 193
 194     my $haenv = $self->{haenv};
 195
 196     if (!$wrote_lrm_status_at_startup) {
 197         if ($self->update_lrm_status()) {
 198             $wrote_lrm_status_at_startup = 1;
 199         } else {
 200             # do nothing
 201             $haenv->sleep(5);
 202             return $self->{shutdown_request} ? 0 : 1;
 203         }
 204     }
 205
 206     my $status = $self->get_local_status();
 207     my $state = $status->{state};
 208
 209     my $ms = $haenv->read_manager_status();
 210     $self->{service_status} =  $ms->{service_status} || {};
 211
 212     my $fence_request = PVE::HA::Tools::count_fenced_services($self->{service_status}, $haenv->nodename());
 213
 214     # do state changes first
 215
 216     my $ctime = $haenv->get_time();
 217
 218     if ($state eq 'wait_for_agent_lock') {
 219
 220         my $service_count = $self->active_service_count();
 221
 222         if (!$fence_request && $service_count && $haenv->quorate()) {
 223             if ($self->get_protected_ha_agent_lock()) {
 224                 $self->set_local_status({ state => 'active' });
 225             }
 226         }
 227
 228     } elsif ($state eq 'lost_agent_lock') {
 229
 230         if (!$fence_request && $haenv->quorate()) {
 231             if ($self->get_protected_ha_agent_lock()) {
 232                 $self->set_local_status({ state => 'active' });
 233             }
 234         }
 235
 236     } elsif ($state eq 'active') {
 237
 238         if ($fence_request) {
 239             $haenv->log('err', "node need to be fenced - releasing agent_lock\n");
 240             $self->set_local_status({ state => 'lost_agent_lock'});
 241         } elsif (!$self->get_protected_ha_agent_lock()) {
 242             $self->set_local_status({ state => 'lost_agent_lock'});
 243         }
 244     }
 245
 246     $status = $self->get_local_status();
 247     $state = $status->{state};
 248
 249     # do work
 250
 251     if ($state eq 'wait_for_agent_lock') {
 252
 253         return 0 if $self->{shutdown_request};
 254
 255         $self->update_lrm_status();
 256
 257         $haenv->sleep(5);
 258
 259     } elsif ($state eq 'active') {
 260
 261         my $startime = $haenv->get_time();
 262
 263         my $max_time = 10;
 264
 265         my $shutdown = 0;
 266
 267         # do work (max_time seconds)
 268         eval {
 269             # fixme: set alert timer
 270
 271             if ($self->{shutdown_request}) {
 272
 273                 if ($self->{mode} eq 'restart') {
 274
 275                     my $service_count = $self->active_service_count();
 276
 277                     if ($service_count == 0) {
 278
 279                         if ($self->run_workers() == 0) {
 280                             if ($self->{ha_agent_wd}) {
 281                                 $haenv->watchdog_close($self->{ha_agent_wd});
 282                                 delete $self->{ha_agent_wd};
 283                             }
 284
 285                             $shutdown = 1;
 286
 287                             # restart with no or freezed services, release the lock
 288                             $haenv->release_ha_agent_lock();
 289                         }
 290                     }
 291                 } else {
 292
 293                     if ($self->run_workers() == 0) {
 294                         if ($self->{shutdown_errors} == 0) {
 295                             if ($self->{ha_agent_wd}) {
 296                                 $haenv->watchdog_close($self->{ha_agent_wd});
 297                                 delete $self->{ha_agent_wd};
 298                             }
 299
 300                             # shutdown with all services stopped thus release the lock
 301                             $haenv->release_ha_agent_lock();
 302                         }
 303
 304                         $shutdown = 1;
 305                     }
 306                 }
 307             } else {
 308
 309                 $self->manage_resources();
 310
 311             }
 312         };
 313         if (my $err = $@) {
 314             $haenv->log('err', "got unexpected error - $err");
 315         }
 316
 317         $self->update_lrm_status();
 318
 319         return 0 if $shutdown;
 320
 321         $haenv->sleep_until($startime + $max_time);
 322
 323     } elsif ($state eq 'lost_agent_lock') {
 324
 325         # Note: watchdog is active an will triger soon!
 326
 327         # so we hope to get the lock back soon!
 328
 329         if ($self->{shutdown_request}) {
 330
 331             my $service_count = $self->active_service_count();
 332
 333             if ($service_count > 0) {
 334                 $haenv->log('err', "get shutdown request in state 'lost_agent_lock' - " .
 335                             "detected $service_count running services");
 336
 337             } else {
 338
 339                 # all services are stopped, so we can close the watchdog
 340
 341                 if ($self->{ha_agent_wd}) {
 342                     $haenv->watchdog_close($self->{ha_agent_wd});
 343                     delete $self->{ha_agent_wd};
 344                 }
 345
 346                 return 0;
 347             }
 348         }
 349
 350         $haenv->sleep(5);
 351
 352     } else {
 353
 354         die "got unexpected status '$state'\n";
 355
 356     }
 357
 358     return 1;
 359 }
 360
 361 sub run_workers {
 362     my ($self) = @_;
 363
 364     my $haenv = $self->{haenv};
 365
 366     my $starttime = $haenv->get_time();
 367
 368     # start workers
 369     my $max_workers = 4;
 370
 371     my $sc = $haenv->read_service_config();
 372
 373     while (($haenv->get_time() - $starttime) < 5) {
 374         my $count =  $self->check_active_workers();
 375
 376         foreach my $sid (keys %{$self->{workers}}) {
 377             last if $count >= $max_workers;
 378             my $w = $self->{workers}->{$sid};
 379             my $cd = $sc->{$sid};
 380             if (!$cd) {
 381                 $haenv->log('err', "missing resource configuration for '$sid'");
 382                 next;
 383             }
 384             if (!$w->{pid}) {
 385                 if ($haenv->can_fork()) {
 386                     my $pid = fork();
 387                     if (!defined($pid)) {
 388                         $haenv->log('err', "fork worker failed");
 389                         $count = 0; last; # abort, try later
 390                     } elsif ($pid == 0) {
 391                         $haenv->after_fork(); # cleanup
 392
 393                         # do work
 394                         my $res = -1;
 395                         eval {
 396                             $res = $self->exec_resource_agent($sid, $cd, $w->{state}, $w->{target});
 397                         };
 398                         if (my $err = $@) {
 399                             $haenv->log('err', $err);
 400                             POSIX::_exit(-1);
 401                         }
 402                         POSIX::_exit($res);
 403                     } else {
 404                         $count++;
 405                         $w->{pid} = $pid;
 406                     }
 407                 } else {
 408                     my $res = -1;
 409                     eval {
 410                         $res = $self->exec_resource_agent($sid, $cd, $w->{state}, $w->{target});
 411                         $res = $res << 8 if $res > 0;
 412                     };
 413                     if (my $err = $@) {
 414                         $haenv->log('err', $err);
 415                     }
 416                     if (defined($w->{uid})) {
 417                         $self->resource_command_finished($sid, $w->{uid}, $res);
 418                     } else {
 419                         $self->stop_command_finished($sid, $res);
 420                     }
 421                 }
 422             }
 423         }
 424
 425         last if !$count;
 426
 427         $haenv->sleep(1);
 428     }
 429
 430     return scalar(keys %{$self->{workers}});
 431 }
 432
 433 sub manage_resources {
 434     my ($self) = @_;
 435
 436     my $haenv = $self->{haenv};
 437
 438     my $nodename = $haenv->nodename();
 439
 440     my $ss = $self->{service_status};
 441
 442     foreach my $sid (keys %$ss) {
 443         my $sd = $ss->{$sid};
 444         next if !$sd->{node};
 445         next if !$sd->{uid};
 446         next if $sd->{node} ne $nodename;
 447         my $req_state = $sd->{state};
 448         next if !defined($req_state);
 449         next if $req_state eq 'freeze';
 450         $self->queue_resource_command($sid, $sd->{uid}, $req_state, $sd->{target});
 451     }
 452
 453     return $self->run_workers();
 454 }
 455
 456 sub queue_resource_command {
 457     my ($self, $sid, $uid, $state, $target) = @_;
 458
 459     if (my $w = $self->{workers}->{$sid}) {
 460         return if $w->{pid}; # already started
 461         # else, delete and overwrite queue entry with new command
 462         delete $self->{workers}->{$sid};
 463     }
 464
 465     $self->{workers}->{$sid} = {
 466         sid => $sid,
 467         uid => $uid,
 468         state => $state,
 469     };
 470
 471     $self->{workers}->{$sid}->{target} = $target if $target;
 472 }
 473
 474 sub check_active_workers {
 475     my ($self) = @_;
 476
 477     # finish/count workers
 478     my $count = 0;
 479     foreach my $sid (keys %{$self->{workers}}) {
 480         my $w = $self->{workers}->{$sid};
 481         if (my $pid = $w->{pid}) {
 482             # check status
 483             my $waitpid = waitpid($pid, WNOHANG);
 484             if (defined($waitpid) && ($waitpid == $pid)) {
 485                 if (defined($w->{uid})) {
 486                     $self->resource_command_finished($sid, $w->{uid}, $?);
 487                 } else {
 488                     $self->stop_command_finished($sid, $?);
 489                 }
 490             } else {
 491                 $count++;
 492             }
 493         }
 494     }
 495
 496     return $count;
 497 }
 498
 499 sub stop_command_finished {
 500     my ($self, $sid, $status) = @_;
 501
 502     my $haenv = $self->{haenv};
 503
 504     my $w = delete $self->{workers}->{$sid};
 505     return if !$w; # should not happen
 506
 507     my $exit_code = -1;
 508
 509     if ($status == -1) {
 510         $haenv->log('err', "resource agent $sid finished - failed to execute");
 511     }  elsif (my $sig = ($status & 127)) {
 512         $haenv->log('err', "resource agent $sid finished - got signal $sig");
 513     } else {
 514         $exit_code = ($status >> 8);
 515     }
 516
 517     if ($exit_code != 0) {
 518         $self->{shutdown_errors}++;
 519     }
 520 }
 521
 522 sub resource_command_finished {
 523     my ($self, $sid, $uid, $status) = @_;
 524
 525     my $haenv = $self->{haenv};
 526
 527     my $w = delete $self->{workers}->{$sid};
 528     return if !$w; # should not happen
 529
 530     my $exit_code = -1;
 531
 532     if ($status == -1) {
 533         $haenv->log('err', "resource agent $sid finished - failed to execute");
 534     }  elsif (my $sig = ($status & 127)) {
 535         $haenv->log('err', "resource agent $sid finished - got signal $sig");
 536     } else {
 537         $exit_code = ($status >> 8);
 538     }
 539
 540     $exit_code = $self->handle_service_exitcode($sid, $w->{state}, $exit_code);
 541
 542     $self->{results}->{$uid} = {
 543         sid => $w->{sid},
 544         state => $w->{state},
 545         exit_code => $exit_code,
 546     };
 547
 548     my $ss = $self->{service_status};
 549
 550     # compute hash of valid/existing uids
 551     my $valid_uids = {};
 552     foreach my $sid (keys %$ss) {
 553         my $sd = $ss->{$sid};
 554         next if !$sd->{uid};
 555         $valid_uids->{$sd->{uid}} = 1;
 556     }
 557
 558     my $results = {};
 559     foreach my $id (keys %{$self->{results}}) {
 560         next if !$valid_uids->{$id};
 561         $results->{$id} = $self->{results}->{$id};
 562     }
 563     $self->{results} = $results;
 564 }
 565
 566 # processes the exit code from a finished resource agent, so that the CRM knows
 567 # if the LRM wants to retry an action based on the current recovery policies for
 568 # the failed service, or the CRM itself must try to recover from the failure.
 569 sub handle_service_exitcode {
 570     my ($self, $sid, $cmd, $exit_code) = @_;
 571
 572     my $haenv = $self->{haenv};
 573     my $tries = $self->{restart_tries};
 574
 575     my $sc = $haenv->read_service_config();
 576     my $cd = $sc->{$sid};
 577
 578     if ($cmd eq 'started') {
 579
 580         if ($exit_code == SUCCESS) {
 581
 582             $tries->{$sid} = 0;
 583
 584             return $exit_code;
 585
 586         } elsif ($exit_code == ERROR) {
 587
 588             $tries->{$sid} = 0 if !defined($tries->{$sid});
 589
 590             $tries->{$sid}++;
 591             if ($tries->{$sid} >= $cd->{max_restart}) {
 592                 $haenv->log('err', "unable to start service $sid on local node".
 593                            " after $tries->{$sid} retries");
 594                 $tries->{$sid} = 0;
 595                 return ERROR;
 596             }
 597
 598             # tell CRM that we retry the start
 599             return ETRY_AGAIN;
 600         }
 601     }
 602
 603     return $exit_code;
 604
 605 }
 606
 607 sub exec_resource_agent {
 608     my ($self, $sid, $service_config, $cmd, @params) = @_;
 609
 610     # setup execution environment
 611
 612     $ENV{'PATH'} = '/sbin:/bin:/usr/sbin:/usr/bin';
 613
 614     my $haenv = $self->{haenv};
 615
 616     my $nodename = $haenv->nodename();
 617
 618     my (undef, $service_type, $service_name) = PVE::HA::Tools::parse_sid($sid);
 619
 620     my $plugin = PVE::HA::Resources->lookup($service_type);
 621     if (!$plugin) {
 622         $haenv->log('err', "service type '$service_type' not implemented");
 623         return EUNKNOWN_SERVICE_TYPE;
 624     }
 625
 626     if ($service_config->{node} ne $nodename) {
 627         $haenv->log('err', "service '$sid' not on this node");
 628         return EWRONG_NODE;
 629     }
 630
 631     my $id = $service_name;
 632
 633     my $running = $plugin->check_running($haenv, $id);
 634
 635     if ($cmd eq 'started') {
 636
 637         return SUCCESS if $running;
 638
 639         $haenv->log("info", "starting service $sid");
 640
 641         $plugin->start($haenv, $id);
 642
 643         $running = $plugin->check_running($haenv, $id);
 644
 645         if ($running) {
 646             $haenv->log("info", "service status $sid started");
 647             return SUCCESS;
 648         } else {
 649             $haenv->log("warning", "unable to start service $sid");
 650             return ERROR;
 651         }
 652
 653     } elsif ($cmd eq 'request_stop' || $cmd eq 'stopped') {
 654
 655         return SUCCESS if !$running;
 656
 657         $haenv->log("info", "stopping service $sid");
 658
 659         $plugin->shutdown($haenv, $id);
 660
 661         $running = $plugin->check_running($haenv, $id);
 662
 663         if (!$running) {
 664             $haenv->log("info", "service status $sid stopped");
 665             return SUCCESS;
 666         } else {
 667             $haenv->log("info", "unable to stop stop service $sid (still running)");
 668             return ERROR;
 669         }
 670
 671     } elsif ($cmd eq 'migrate' || $cmd eq 'relocate') {
 672
 673         my $target = $params[0];
 674         if (!defined($target)) {
 675             die "$cmd '$sid' failed - missing target\n" if !defined($target);
 676             return EINVALID_PARAMETER;
 677         }
 678
 679         if ($service_config->{node} eq $target) {
 680             # already there
 681             return SUCCESS;
 682         }
 683
 684         my $online = ($cmd eq 'migrate') ? 1 : 0;
 685
 686         my $res = $plugin->migrate($haenv, $id, $target, $online);
 687
 688         # something went wrong if service is still on this node
 689         if (!$res) {
 690             $haenv->log("err", "service $sid not moved (migration error)");
 691             return ERROR;
 692         }
 693
 694         return SUCCESS;
 695
 696     } elsif ($cmd eq 'error') {
 697
 698         if ($running) {
 699             $haenv->log("err", "service $sid is in an error state while running");
 700         } else {
 701             $haenv->log("warning", "service $sid is not running and in an error state");
 702         }
 703         return SUCCESS; # error always succeeds
 704
 705     }
 706
 707     $haenv->log("err", "implement me (cmd '$cmd')");
 708     return EUNKNOWN_COMMAND;
 709 }
 710
 711
 712 1;