src/PVE/HA/Manager.pm

   1 package PVE::HA::Manager;
   2
   3 use strict;
   4 use warnings;
   5 use Digest::MD5 qw(md5_base64);
   6
   7 use Data::Dumper;
   8 use PVE::Tools;
   9 use PVE::HA::Tools ':exit_codes';
  10 use PVE::HA::NodeStatus;
  11
  12 my $fence_delay = 60;
  13
  14 sub new {
  15     my ($this, $haenv) = @_;
  16
  17     my $class = ref($this) || $this;
  18
  19     my $ms = $haenv->read_manager_status();
  20
  21     $ms->{master_node} = $haenv->nodename();
  22
  23     my $ns = PVE::HA::NodeStatus->new($haenv, $ms->{node_status} || {});
  24
  25     # fixme: use separate class  PVE::HA::ServiceStatus
  26     my $ss = $ms->{service_status} || {};
  27
  28     my $self = bless {
  29         haenv => $haenv,
  30         ms => $ms, # master status
  31         ns => $ns, # PVE::HA::NodeStatus
  32         ss => $ss, # service status
  33     }, $class;
  34
  35     return $self;
  36 }
  37
  38 sub cleanup {
  39     my ($self) = @_;
  40
  41     # todo: ?
  42 }
  43
  44 sub flush_master_status {
  45     my ($self) = @_;
  46
  47     my ($haenv, $ms, $ns, $ss) = ($self->{haenv}, $self->{ms}, $self->{ns}, $self->{ss});
  48
  49     $ms->{node_status} = $ns->{status};
  50     $ms->{service_status} = $ss;
  51     $ms->{timestamp} = $haenv->get_time();
  52
  53     $haenv->write_manager_status($ms);
  54 }
  55
  56 sub select_service_node {
  57     my ($groups, $online_node_usage, $service_conf, $current_node, $try_next) = @_;
  58
  59     my $group = { 'nodes' => { $service_conf->{node} => 1 } }; # default group
  60
  61     $group =  $groups->{ids}->{$service_conf->{group}} if $service_conf->{group} &&
  62         $groups->{ids}->{$service_conf->{group}};
  63
  64     my $pri_groups = {};
  65     my $group_members = {};
  66     foreach my $entry (keys %{$group->{nodes}}) {
  67         my ($node, $pri) = ($entry, 0);
  68         if ($entry =~ m/^(\S+):(\d+)$/) {
  69             ($node, $pri) = ($1, $2);
  70         }
  71         next if !defined($online_node_usage->{$node}); # offline
  72         $pri_groups->{$pri}->{$node} = 1;
  73         $group_members->{$node} = $pri;
  74     }
  75
  76
  77     # add non-group members to unrestricted groups (priority -1)
  78     if (!$group->{restricted}) {
  79         my $pri = -1;
  80         foreach my $node (keys %$online_node_usage) {
  81             next if defined($group_members->{$node});
  82             $pri_groups->{$pri}->{$node} = 1;
  83             $group_members->{$node} = -1;
  84         }
  85     }
  86
  87
  88     my @pri_list = sort {$b <=> $a} keys %$pri_groups;
  89     return undef if !scalar(@pri_list);
  90
  91     if (!$try_next && $group->{nofailback} && defined($group_members->{$current_node})) {
  92         return $current_node;
  93     }
  94
  95     # select node from top priority node list
  96
  97     my $top_pri = $pri_list[0];
  98
  99     my @nodes = sort {
 100         $online_node_usage->{$a} <=> $online_node_usage->{$b} || $a cmp $b
 101     } keys %{$pri_groups->{$top_pri}};
 102
 103     my $found;
 104     for (my $i = scalar(@nodes) - 1; $i >= 0; $i--) {
 105         my $node = $nodes[$i];
 106         if ($node eq $current_node) {
 107             $found = $i;
 108             last;
 109         }
 110     }
 111
 112     if ($try_next) {
 113
 114         if (defined($found) && ($found < (scalar(@nodes) - 1))) {
 115             return $nodes[$found + 1];
 116         } else {
 117             return $nodes[0];
 118         }
 119
 120     } else {
 121
 122         return $nodes[$found] if defined($found);
 123
 124         return $nodes[0];
 125
 126     }
 127 }
 128
 129 my $uid_counter = 0;
 130
 131 sub compute_new_uuid {
 132     my ($state) = @_;
 133
 134     $uid_counter++;
 135     return md5_base64($state . $$ . time() . $uid_counter);
 136 }
 137
 138 my $valid_service_states = {
 139     stopped => 1,
 140     request_stop => 1,
 141     started => 1,
 142     fence => 1,
 143     migrate => 1,
 144     relocate => 1,
 145     freeze => 1,
 146     error => 1,
 147 };
 148
 149 sub recompute_online_node_usage {
 150     my ($self) = @_;
 151
 152     my $online_node_usage = {};
 153
 154     my $online_nodes = $self->{ns}->list_online_nodes();
 155
 156     foreach my $node (@$online_nodes) {
 157         $online_node_usage->{$node} = 0;
 158     }
 159
 160     foreach my $sid (keys %{$self->{ss}}) {
 161         my $sd = $self->{ss}->{$sid};
 162         my $state = $sd->{state};
 163         if (defined($online_node_usage->{$sd->{node}})) {
 164             if (($state eq 'started') || ($state eq 'request_stop') ||
 165                 ($state eq 'fence') || ($state eq 'freeze') || ($state eq 'error')) {
 166                 $online_node_usage->{$sd->{node}}++;
 167             } elsif (($state eq 'migrate') || ($state eq 'relocate')) {
 168                 $online_node_usage->{$sd->{target}}++;
 169             } elsif ($state eq 'stopped') {
 170                 # do nothing
 171             } else {
 172                 die "should not be reached";
 173             }
 174         }
 175     }
 176
 177     $self->{online_node_usage} = $online_node_usage;
 178 }
 179
 180 my $change_service_state = sub {
 181     my ($self, $sid, $new_state, %params) = @_;
 182
 183     my ($haenv, $ss) = ($self->{haenv}, $self->{ss});
 184
 185     my $sd = $ss->{$sid} || die "no such service '$sid";
 186
 187     my $old_state = $sd->{state};
 188     my $old_node = $sd->{node};
 189
 190     die "no state change" if $old_state eq $new_state; # just to be sure
 191
 192     die "invalid CRM service state '$new_state'\n" if !$valid_service_states->{$new_state};
 193
 194     foreach my $k (keys %$sd) { delete $sd->{$k}; };
 195
 196     $sd->{state} = $new_state;
 197     $sd->{node} = $old_node;
 198
 199     my $text_state = '';
 200     foreach my $k (sort keys %params) {
 201         my $v = $params{$k};
 202         $text_state .= ", " if $text_state;
 203         $text_state .= "$k = $v";
 204         $sd->{$k} = $v;
 205     }
 206
 207     $self->recompute_online_node_usage();
 208
 209     $sd->{uid} = compute_new_uuid($new_state);
 210
 211     $text_state = "  ($text_state)" if $text_state;
 212     $haenv->log('info', "service '$sid': state changed from '${old_state}'" .
 213                 " to '${new_state}'$text_state");
 214 };
 215
 216 # after a node was fenced this recovers the service to a new node
 217 my $recover_fenced_service = sub {
 218     my ($self, $sid, $cd) = @_;
 219
 220     my ($haenv, $ss) = ($self->{haenv}, $self->{ss});
 221
 222     my $sd = $ss->{$sid};
 223
 224     if ($sd->{state} ne 'fence') { # should not happen
 225         $haenv->log('err', "cannot recover service '$sid' from fencing," .
 226                     " wrong state '$sd->{state}'");
 227         return;
 228     }
 229
 230     my $fenced_node = $sd->{node}; # for logging purpose
 231
 232     $self->recompute_online_node_usage(); # we want the most current node state
 233
 234     my $recovery_node = select_service_node($self->{groups},
 235                                             $self->{online_node_usage},
 236                                             $cd, $sd->{node});
 237
 238     if ($recovery_node) {
 239         $haenv->log('info', "recover service '$sid' from fenced node " .
 240                     "'$fenced_node' to node '$recovery_node'");
 241
 242         $haenv->steal_service($sid, $sd->{node}, $recovery_node);
 243
 244         # $sd *is normally read-only*, fencing is the exception
 245         $cd->{node} = $sd->{node} = $recovery_node;
 246         &$change_service_state($self, $sid, 'started', node => $recovery_node);
 247     } else {
 248         # no node found, let the service in 'fence' state and try again
 249         $haenv->log('err', "recovering service '$sid' from fenced node " .
 250                     "'$fenced_node' failed, no recovery node found");
 251     }
 252 };
 253
 254 # read LRM status for all nodes
 255 sub read_lrm_status {
 256     my ($self) = @_;
 257
 258     my $nodes = $self->{ns}->list_nodes();
 259     my $haenv = $self->{haenv};
 260
 261     my $results = {};
 262     my $modes = {};
 263     foreach my $node (@$nodes) {
 264         my $lrm_status = $haenv->read_lrm_status($node);
 265         $modes->{$node} = $lrm_status->{mode} || 'active';
 266         foreach my $uid (keys %{$lrm_status->{results}}) {
 267             next if $results->{$uid}; # should not happen
 268             $results->{$uid} = $lrm_status->{results}->{$uid};
 269         }
 270     }
 271
 272
 273     return ($results, $modes);
 274 }
 275
 276 # read new crm commands and save them into crm master status
 277 sub update_crm_commands {
 278     my ($self) = @_;
 279
 280     my ($haenv, $ms, $ns, $ss) = ($self->{haenv}, $self->{ms}, $self->{ns}, $self->{ss});
 281
 282     my $cmdlist = $haenv->read_crm_commands();
 283
 284     foreach my $cmd (split(/\n/, $cmdlist)) {
 285         chomp $cmd;
 286
 287         if ($cmd =~ m/^(migrate|relocate)\s+(\S+)\s+(\S+)$/) {
 288             my ($task, $sid, $node) = ($1, $2, $3);
 289             if (my $sd = $ss->{$sid}) {
 290                 if (!$ns->node_is_online($node)) {
 291                     $haenv->log('err', "crm command error - node not online: $cmd");
 292                 } else {
 293                     if ($node eq $sd->{node}) {
 294                         $haenv->log('info', "ignore crm command - service already on target node: $cmd");
 295                     } else {
 296                         $haenv->log('info', "got crm command: $cmd");
 297                         $ss->{$sid}->{cmd} = [ $task, $node];
 298                     }
 299                 }
 300             } else {
 301                 $haenv->log('err', "crm command error - no such service: $cmd");
 302             }
 303
 304         } else {
 305             $haenv->log('err', "unable to parse crm command: $cmd");
 306         }
 307     }
 308
 309 }
 310
 311 sub manage {
 312     my ($self) = @_;
 313
 314     my ($haenv, $ms, $ns, $ss) = ($self->{haenv}, $self->{ms}, $self->{ns}, $self->{ss});
 315
 316     $ns->update($haenv->get_node_info());
 317
 318     if (!$ns->node_is_online($haenv->nodename())) {
 319         $haenv->log('info', "master seems offline");
 320         return;
 321     }
 322
 323     my ($lrm_results, $lrm_modes) = $self->read_lrm_status();
 324
 325     my $sc = $haenv->read_service_config();
 326
 327     $self->{groups} = $haenv->read_group_config(); # update
 328
 329     # compute new service status
 330
 331     # add new service
 332     foreach my $sid (sort keys %$sc) {
 333         next if $ss->{$sid}; # already there
 334         $haenv->log('info', "adding new service '$sid' on node '$sc->{$sid}->{node}'");
 335         # assume we are running to avoid relocate running service at add
 336         $ss->{$sid} = { state => 'started', node => $sc->{$sid}->{node},
 337                         uid => compute_new_uuid('started') };
 338     }
 339
 340     # remove stale service from manager state
 341     foreach my $sid (keys %$ss) {
 342         next if $sc->{$sid};
 343         $haenv->log('info', "removing stale service '$sid' (no config)");
 344         delete $ss->{$sid};
 345     }
 346
 347     $self->update_crm_commands();
 348
 349     for (;;) {
 350         my $repeat = 0;
 351
 352         $self->recompute_online_node_usage();
 353
 354         foreach my $sid (keys %$ss) {
 355             my $sd = $ss->{$sid};
 356             my $cd = $sc->{$sid} || { state => 'disabled' };
 357
 358             my $lrm_res = $sd->{uid} ? $lrm_results->{$sd->{uid}} : undef;
 359
 360             my $last_state = $sd->{state};
 361
 362             if ($last_state eq 'stopped') {
 363
 364                 $self->next_state_stopped($sid, $cd, $sd, $lrm_res);
 365
 366             } elsif ($last_state eq 'started') {
 367
 368                 $self->next_state_started($sid, $cd, $sd, $lrm_res);
 369
 370             } elsif ($last_state eq 'migrate' || $last_state eq 'relocate') {
 371
 372                 $self->next_state_migrate_relocate($sid, $cd, $sd, $lrm_res);
 373
 374             } elsif ($last_state eq 'fence') {
 375
 376                 # do nothing here - wait until fenced
 377
 378             } elsif ($last_state eq 'request_stop') {
 379
 380                 $self->next_state_request_stop($sid, $cd, $sd, $lrm_res);
 381
 382             } elsif ($last_state eq 'freeze') {
 383
 384                 my $lrm_mode = $sd->{node} ? $lrm_modes->{$sd->{node}} : undef;
 385                 # unfreeze
 386                 &$change_service_state($self, $sid, 'started')
 387                     if $lrm_mode && $lrm_mode eq 'active';
 388
 389             } elsif ($last_state eq 'error') {
 390
 391                 $self->next_state_error($sid, $cd, $sd, $lrm_res);
 392
 393             } else {
 394
 395                 die "unknown service state '$last_state'";
 396             }
 397
 398             my $lrm_mode = $sd->{node} ? $lrm_modes->{$sd->{node}} : undef;
 399             if ($lrm_mode && $lrm_mode eq 'restart') {
 400                 if (($sd->{state} eq 'started' || $sd->{state} eq 'stopped' ||
 401                      $sd->{state} eq 'request_stop')) {
 402                     &$change_service_state($self, $sid, 'freeze');
 403                 }
 404             }
 405
 406             $repeat = 1 if $sd->{state} ne $last_state;
 407         }
 408
 409         # handle fencing
 410         my $fenced_nodes = {};
 411         foreach my $sid (keys %$ss) {
 412             my $sd = $ss->{$sid};
 413             next if $sd->{state} ne 'fence';
 414
 415             if (!defined($fenced_nodes->{$sd->{node}})) {
 416                 $fenced_nodes->{$sd->{node}} = $ns->fence_node($sd->{node}) || 0;
 417             }
 418
 419             next if !$fenced_nodes->{$sd->{node}};
 420
 421             # node fence was successful - recover service
 422             &$recover_fenced_service($self, $sid, $sc->{$sid});
 423         }
 424
 425         last if !$repeat;
 426     }
 427
 428     $self->flush_master_status();
 429 }
 430
 431 # functions to compute next service states
 432 # $cd: service configuration data (read only)
 433 # $sd: service status data (read only)
 434 #
 435 # Note: use change_service_state() to alter state
 436 #
 437
 438 sub next_state_request_stop {
 439     my ($self, $sid, $cd, $sd, $lrm_res) = @_;
 440
 441     my $haenv = $self->{haenv};
 442     my $ns = $self->{ns};
 443
 444     # check result from LRM daemon
 445     if ($lrm_res) {
 446         my $exit_code = $lrm_res->{exit_code};
 447         if ($exit_code == SUCCESS) {
 448             &$change_service_state($self, $sid, 'stopped');
 449             return;
 450         } else {
 451             $haenv->log('err', "service '$sid' stop failed (exit code $exit_code)");
 452             &$change_service_state($self, $sid, 'error'); # fixme: what state?
 453             return;
 454         }
 455     }
 456
 457     if ($ns->node_is_offline_delayed($sd->{node}, $fence_delay)) {
 458         &$change_service_state($self, $sid, 'fence');
 459         return;
 460     }
 461 }
 462
 463 sub next_state_migrate_relocate {
 464     my ($self, $sid, $cd, $sd, $lrm_res) = @_;
 465
 466     my $haenv = $self->{haenv};
 467     my $ns = $self->{ns};
 468
 469     # check result from LRM daemon
 470     if ($lrm_res) {
 471         my $exit_code = $lrm_res->{exit_code};
 472         my $req_state = $cd->{state} eq 'enabled' ? 'started' : 'request_stop';
 473         if ($exit_code == SUCCESS) {
 474             &$change_service_state($self, $sid, $req_state, node => $sd->{target});
 475             return;
 476         } else {
 477             $haenv->log('err', "service '$sid' - migration failed (exit code $exit_code)");
 478             &$change_service_state($self, $sid, $req_state, node => $sd->{node});
 479             return;
 480         }
 481     }
 482
 483     if ($ns->node_is_offline_delayed($sd->{node}, $fence_delay)) {
 484         &$change_service_state($self, $sid, 'fence');
 485         return;
 486     }
 487 }
 488
 489
 490 sub next_state_stopped {
 491     my ($self, $sid, $cd, $sd, $lrm_res) = @_;
 492
 493     my $haenv = $self->{haenv};
 494     my $ns = $self->{ns};
 495
 496     if ($sd->{node} ne $cd->{node}) {
 497         # this can happen if we fence a node with active migrations
 498         # hack: modify $sd (normally this should be considered read-only)
 499         $haenv->log('info', "fixup service '$sid' location ($sd->{node} => $cd->{node})");
 500         $sd->{node} = $cd->{node};
 501     }
 502
 503     if ($sd->{cmd}) {
 504         my ($cmd, $target) = @{$sd->{cmd}};
 505         delete $sd->{cmd};
 506
 507         if ($cmd eq 'migrate' || $cmd eq 'relocate') {
 508             if (!$ns->node_is_online($target)) {
 509                 $haenv->log('err', "ignore service '$sid' $cmd request - node '$target' not online");
 510             } elsif ($sd->{node} eq $target) {
 511                 $haenv->log('info', "ignore service '$sid' $cmd request - service already on node '$target'");
 512             } else {
 513                 &$change_service_state($self, $sid, $cmd, node => $sd->{node},
 514                                        target => $target);
 515                 return;
 516             }
 517         } else {
 518             $haenv->log('err', "unknown command '$cmd' for service '$sid'");
 519         }
 520     }
 521
 522     if ($cd->{state} eq 'disabled') {
 523         # NOTE: do nothing here, the stop state is an exception as we do not
 524         # process the LRM result here, thus the LRM always tries to stop the
 525         # service (protection for the case no CRM is active)
 526         return;
 527     }
 528
 529     if ($cd->{state} eq 'enabled') {
 530         # simply mark it started, if it's on the wrong node
 531         # next_state_started will fix that for us
 532         &$change_service_state($self, $sid, 'started', node => $sd->{node});
 533         return;
 534     }
 535
 536     $haenv->log('err', "service '$sid' - unknown state '$cd->{state}' in service configuration");
 537 }
 538
 539 sub next_state_started {
 540     my ($self, $sid, $cd, $sd, $lrm_res) = @_;
 541
 542     my $haenv = $self->{haenv};
 543     my $master_status = $self->{ms};
 544     my $ns = $self->{ns};
 545
 546     if (!$ns->node_is_online($sd->{node})) {
 547         if ($ns->node_is_offline_delayed($sd->{node}, $fence_delay)) {
 548             &$change_service_state($self, $sid, 'fence');
 549         }
 550         return;
 551     }
 552
 553     if ($cd->{state} eq 'disabled') {
 554         &$change_service_state($self, $sid, 'request_stop');
 555         return;
 556     }
 557
 558     if ($cd->{state} eq 'enabled') {
 559
 560         if ($sd->{cmd}) {
 561             my ($cmd, $target) = @{$sd->{cmd}};
 562             delete $sd->{cmd};
 563
 564             if ($cmd eq 'migrate' || $cmd eq 'relocate') {
 565                 if (!$ns->node_is_online($target)) {
 566                     $haenv->log('err', "ignore service '$sid' $cmd request - node '$target' not online");
 567                 } elsif ($sd->{node} eq $target) {
 568                     $haenv->log('info', "ignore service '$sid' $cmd request - service already on node '$target'");
 569                 } else {
 570                     $haenv->log('info', "$cmd service '$sid' to node '$target'");
 571                     &$change_service_state($self, $sid, $cmd, node => $sd->{node}, target => $target);
 572                 }
 573             } else {
 574                 $haenv->log('err', "unknown command '$cmd' for service '$sid'");
 575             }
 576         } else {
 577
 578             my $try_next = 0;
 579             if ($lrm_res) {
 580                 my $ec = $lrm_res->{exit_code};
 581                 if ($ec == SUCCESS) {
 582
 583                     $master_status->{relocate_trial}->{$sid} = 0;
 584
 585                 } elsif ($ec == ERROR) {
 586                     # apply our relocate policy if we got ERROR from the LRM
 587
 588                     my $try = $master_status->{relocate_trial}->{$sid} || 0;
 589
 590                     if ($try < $cd->{max_relocate}) {
 591
 592                         $try++;
 593                         # tell select_service_node to relocate if possible
 594                         $try_next = 1;
 595
 596                         $haenv->log('warning', "starting service $sid on node".
 597                                    " '$sd->{node}' failed, relocating service.");
 598                         $master_status->{relocate_trial}->{$sid} = $try;
 599
 600                     } else {
 601
 602                         $haenv->log('err', "recovery policy for service".
 603                                    " $sid failed, entering error state!");
 604                         &$change_service_state($self, $sid, 'error');
 605                         return;
 606
 607                     }
 608                 } else {
 609                     $haenv->log('err', "service '$sid' got unrecoverable error" .
 610                                 " (exit code $ec))");
 611                     # we have no save way out (yet) for other errors
 612                     &$change_service_state($self, $sid, 'error');
 613                     return;
 614                 }
 615             }
 616
 617             my $node = select_service_node($self->{groups}, $self->{online_node_usage},
 618                                            $cd, $sd->{node}, $try_next);
 619
 620             if ($node && ($sd->{node} ne $node)) {
 621                 if ($cd->{type} eq 'vm') {
 622                     $haenv->log('info', "migrate service '$sid' to node '$node' (running)");
 623                     &$change_service_state($self, $sid, 'migrate', node => $sd->{node}, target => $node);
 624                 } else {
 625                     $haenv->log('info', "relocate service '$sid' to node '$node'");
 626                     &$change_service_state($self, $sid, 'relocate', node => $sd->{node}, target => $node);
 627                 }
 628             } else {
 629                 # ensure service get started again if it went unexpected down
 630                 $sd->{uid} = compute_new_uuid($sd->{state});
 631             }
 632         }
 633
 634         return;
 635     }
 636
 637     $haenv->log('err', "service '$sid' - unknown state '$cd->{state}' in service configuration");
 638 }
 639
 640 sub next_state_error {
 641     my ($self, $sid, $cd, $sd, $lrm_res) = @_;
 642
 643     my $ns = $self->{ns};
 644
 645     if ($cd->{state} eq 'disabled') {
 646         &$change_service_state($self, $sid, 'stopped');
 647         return;
 648     }
 649
 650     if ($ns->node_is_offline_delayed($sd->{node}, $fence_delay)) {
 651         &$change_service_state($self, $sid, 'fence');
 652         return;
 653     }
 654
 655 }
 656
 657 1;