src/PVE/HA/Manager.pm

   1 package PVE::HA::Manager;
   2
   3 use strict;
   4 use warnings;
   5 use Digest::MD5 qw(md5_base64);
   6
   7 use Data::Dumper;
   8
   9 use PVE::HA::NodeStatus;
  10
  11 sub new {
  12     my ($this, $haenv) = @_;
  13
  14     my $class = ref($this) || $this;
  15
  16     my $ms = $haenv->read_manager_status();
  17
  18     $ms->{master_node} = $haenv->nodename();
  19
  20     my $ns = PVE::HA::NodeStatus->new($haenv, $ms->{node_status} || {});
  21
  22     # fixme: use separate class  PVE::HA::ServiceStatus
  23     my $ss = $ms->{service_status} || {};
  24
  25     my $self = bless {
  26         haenv => $haenv,
  27         ms => $ms, # master status
  28         ns => $ns, # PVE::HA::NodeStatus
  29         ss => $ss, # service status
  30     }, $class;
  31
  32     return $self;
  33 }
  34
  35 sub cleanup {
  36     my ($self) = @_;
  37
  38     # todo: ?
  39 }
  40
  41 sub flush_master_status {
  42     my ($self) = @_;
  43
  44     my ($haenv, $ms, $ns, $ss) = ($self->{haenv}, $self->{ms}, $self->{ns}, $self->{ss});
  45
  46     $ms->{node_status} = $ns->{status};
  47     $ms->{service_status} = $ss;
  48
  49     $haenv->write_manager_status($ms);
  50 }
  51
  52 sub select_service_node {
  53     my ($groups, $online_node_usage, $service_conf, $current_node, $try_next) = @_;
  54
  55     my $group = { 'nodes' => $service_conf->{node} }; # default group
  56
  57     $group =  $groups->{ids}->{$service_conf->{group}} if $service_conf->{group} &&
  58         $groups->{ids}->{$service_conf->{group}};
  59
  60     my $pri_groups = {};
  61     my $group_members = {};
  62     foreach my $entry (PVE::Tools::split_list($group->{nodes})) {
  63         my ($node, $pri) = ($entry, 0);
  64         if ($entry =~ m/^(\S+):(\d+)$/) {
  65             ($node, $pri) = ($1, $2);
  66         }
  67         next if !defined($online_node_usage->{$node}); # offline
  68         $pri_groups->{$pri}->{$node} = 1;
  69         $group_members->{$node} = $pri;
  70     }
  71
  72
  73     # add non-group members to unrestricted groups (priority -1)
  74     if (!$group->{restricted}) {
  75         my $pri = -1;
  76         foreach my $node (keys %$online_node_usage) {
  77             next if defined($group_members->{$node});
  78             $pri_groups->{$pri}->{$node} = 1;
  79             $group_members->{$node} = -1;
  80         }
  81     }
  82
  83
  84     my @pri_list = sort {$b <=> $a} keys %$pri_groups;
  85     return undef if !scalar(@pri_list);
  86
  87     if (!$try_next && $group->{nofailback} && defined($group_members->{$current_node})) {
  88         return $current_node;
  89     }
  90
  91     # select node from top priority node list
  92
  93     my $top_pri = $pri_list[0];
  94
  95     my @nodes = sort { $online_node_usage->{$a} <=> $online_node_usage->{$b} } keys %{$pri_groups->{$top_pri}};
  96
  97     my $found;
  98     for (my $i = scalar(@nodes) - 1; $i >= 0; $i--) {
  99         my $node = $nodes[$i];
 100         if ($node eq $current_node) {
 101             $found = $i;
 102             last;
 103         }
 104     }
 105
 106     if ($try_next) {
 107
 108         if (defined($found) && ($found < (scalar(@nodes) - 1))) {
 109             return $nodes[$found + 1];
 110         } else {
 111             return $nodes[0];
 112         }
 113
 114     } else {
 115
 116         return $nodes[$found] if defined($found);
 117
 118         return $nodes[0];
 119
 120     }
 121 }
 122
 123 my $uid_counter = 0;
 124
 125 my $valid_service_states = {
 126     stopped => 1,
 127     request_stop => 1,
 128     started => 1,
 129     fence => 1,
 130     migrate => 1,
 131     relocate => 1,
 132     error => 1,
 133 };
 134
 135 sub recompute_online_node_usage {
 136     my ($self) = @_;
 137
 138     my $online_node_usage = {};
 139
 140     my $online_nodes = $self->{ns}->list_online_nodes();
 141
 142     foreach my $node (@$online_nodes) {
 143         $online_node_usage->{$node} = 0;
 144     }
 145
 146     foreach my $sid (keys %{$self->{ss}}) {
 147         my $sd = $self->{ss}->{$sid};
 148         my $state = $sd->{state};
 149         if (defined($online_node_usage->{$sd->{node}})) {
 150             if (($state eq 'started') || ($state eq 'request_stop') ||
 151                 ($state eq 'fence') || ($state eq 'error')) {
 152                 $online_node_usage->{$sd->{node}}++;
 153             } elsif (($state eq 'migrate') || ($state eq 'relocate')) {
 154                 $online_node_usage->{$sd->{target}}++;
 155             } elsif ($state eq 'stopped') {
 156                 # do nothing
 157             } else {
 158                 die "should not be reached";
 159             }
 160         }
 161     }
 162
 163     $self->{online_node_usage} = $online_node_usage;
 164 }
 165
 166 my $change_service_state = sub {
 167     my ($self, $sid, $new_state, %params) = @_;
 168
 169     my ($haenv, $ss) = ($self->{haenv}, $self->{ss});
 170
 171     my $sd = $ss->{$sid} || die "no such service '$sid";
 172
 173     my $old_state = $sd->{state};
 174     my $old_node = $sd->{node};
 175
 176     die "no state change" if $old_state eq $new_state; # just to be sure
 177
 178     die "invalid CRM service state '$new_state'\n" if !$valid_service_states->{$new_state};
 179
 180     foreach my $k (keys %$sd) { delete $sd->{$k}; };
 181
 182     $sd->{state} = $new_state;
 183     $sd->{node} = $old_node;
 184
 185     my $text_state = '';
 186     foreach my $k (keys %params) {
 187         my $v = $params{$k};
 188         $text_state .= ", " if $text_state;
 189         $text_state .= "$k = $v";
 190         $sd->{$k} = $v;
 191     }
 192
 193     $self->recompute_online_node_usage();
 194
 195     $uid_counter++;
 196     $sd->{uid} = md5_base64($new_state . $$ . time() . $uid_counter);
 197
 198     $text_state = " ($text_state)" if $text_state;
 199     $haenv->log('info', "service '$sid': state changed from '${old_state}' to '${new_state}' $text_state\n");
 200 };
 201
 202 # read LRM status for all active nodes
 203 sub read_lrm_status {
 204     my ($self) = @_;
 205
 206     my $nodes = $self->{ns}->list_online_nodes();
 207     my $haenv = $self->{haenv};
 208
 209     my $res = {};
 210
 211     foreach my $node (@$nodes) {
 212         my $ls = $haenv->read_lrm_status($node);
 213         foreach my $uid (keys %$ls) {
 214             next if $res->{$uid}; # should not happen
 215             $res->{$uid} = $ls->{$uid};
 216         }
 217     }
 218
 219     return $res;
 220 }
 221
 222 # read new crm commands and save them into crm master status
 223 sub update_crm_commands {
 224     my ($self) = @_;
 225
 226     my ($haenv, $ms, $ns, $ss) = ($self->{haenv}, $self->{ms}, $self->{ns}, $self->{ss});
 227
 228     my $cmdlist = $haenv->read_crm_commands();
 229
 230     foreach my $cmd (split(/\n/, $cmdlist)) {
 231         chomp $cmd;
 232
 233         if ($cmd =~ m/^(migrate|relocate)\s+(\S+)\s+(\S+)$/) {
 234             my ($task, $sid, $node) = ($1, $2, $3);
 235             if (my $sd = $ss->{$sid}) {
 236                 if (!$ns->node_is_online($node)) {
 237                     $haenv->log('err', "crm command error - node not online: $cmd");
 238                 } else {
 239                     if ($node eq $sd->{node}) {
 240                         $haenv->log('info', "ignore crm command - service already on target node: $cmd");
 241                     } else {
 242                         $haenv->log('info', "got crm command: $cmd");
 243                         $ss->{$sid}->{cmd} = [ $task, $node];
 244                     }
 245                 }
 246             } else {
 247                 $haenv->log('err', "crm command error - no such service: $cmd");
 248             }
 249
 250         } else {
 251             $haenv->log('err', "unable to parse crm command: $cmd");
 252         }
 253     }
 254
 255 }
 256
 257 sub manage {
 258     my ($self) = @_;
 259
 260     my ($haenv, $ms, $ns, $ss) = ($self->{haenv}, $self->{ms}, $self->{ns}, $self->{ss});
 261
 262     $ns->update($haenv->get_node_info());
 263
 264     if (!$ns->node_is_online($haenv->nodename())) {
 265         $haenv->log('info', "master seems offline\n");
 266         return;
 267     }
 268
 269     my $lrm_status = $self->read_lrm_status();
 270
 271     my $sc = $haenv->read_service_config();
 272
 273     $self->{groups} = $haenv->read_group_config(); # update
 274
 275     # compute new service status
 276
 277     # add new service
 278     foreach my $sid (keys %$sc) {
 279         next if $ss->{$sid}; # already there
 280         $haenv->log('info', "Adding new service '$sid'\n");
 281         # assume we are running to avoid relocate running service at add
 282         $ss->{$sid} = { state => 'started', node => $sc->{$sid}->{node}};
 283     }
 284
 285     $self->update_crm_commands();
 286
 287     for (;;) {
 288         my $repeat = 0;
 289
 290         $self->recompute_online_node_usage();
 291
 292         foreach my $sid (keys %$ss) {
 293             my $sd = $ss->{$sid};
 294             my $cd = $sc->{$sid} || { state => 'disabled' };
 295
 296             my $lrm_res = $sd->{uid} ? $lrm_status->{$sd->{uid}} : undef;
 297
 298             my $last_state = $sd->{state};
 299
 300             if ($last_state eq 'stopped') {
 301
 302                 $self->next_state_stopped($sid, $cd, $sd, $lrm_res);
 303
 304             } elsif ($last_state eq 'started') {
 305
 306                 $self->next_state_started($sid, $cd, $sd, $lrm_res);
 307
 308             } elsif ($last_state eq 'migrate' || $last_state eq 'relocate') {
 309
 310                 $self->next_state_migrate_relocate($sid, $cd, $sd, $lrm_res);
 311
 312             } elsif ($last_state eq 'fence') {
 313
 314                 # do nothing here - wait until fenced
 315
 316             } elsif ($last_state eq 'request_stop') {
 317
 318                 $self->next_state_request_stop($sid, $cd, $sd, $lrm_res);
 319
 320             } elsif ($last_state eq 'error') {
 321
 322                 # fixme:
 323
 324             } else {
 325
 326                 die "unknown service state '$last_state'";
 327             }
 328
 329             $repeat = 1 if $sd->{state} ne $last_state;
 330         }
 331
 332         # handle fencing
 333         my $fenced_nodes = {};
 334         foreach my $sid (keys %$ss) {
 335             my $sd = $ss->{$sid};
 336             next if $sd->{state} ne 'fence';
 337
 338             if (!defined($fenced_nodes->{$sd->{node}})) {
 339                 $fenced_nodes->{$sd->{node}} = $ns->fence_node($sd->{node}) || 0;
 340             }
 341
 342             next if !$fenced_nodes->{$sd->{node}};
 343
 344             # node fence was sucessful - mark service as stopped
 345             &$change_service_state($self, $sid, 'stopped');
 346         }
 347
 348         last if !$repeat;
 349     }
 350
 351     # remove stale services
 352     # fixme:
 353
 354     $self->flush_master_status();
 355 }
 356
 357 # functions to compute next service states
 358 # $cd: service configuration data (read only)
 359 # $sd: service status data (read only)
 360 #
 361 # Note: use change_service_state() to alter state
 362 #
 363
 364 sub next_state_request_stop {
 365     my ($self, $sid, $cd, $sd, $lrm_res) = @_;
 366
 367     my $haenv = $self->{haenv};
 368     my $ns = $self->{ns};
 369
 370     # check result from LRM daemon
 371     if ($lrm_res) {
 372         my $exit_code = $lrm_res->{exit_code};
 373         if ($exit_code == 0) {
 374             &$change_service_state($self, $sid, 'stopped');
 375             return;
 376         } else {
 377             &$change_service_state($self, $sid, 'error'); # fixme: what state?
 378             return;
 379         }
 380     }
 381
 382     if (!$ns->node_is_online($sd->{node})) {
 383         &$change_service_state($self, $sid, 'fence');
 384         return;
 385     }
 386 }
 387
 388 sub next_state_migrate_relocate {
 389     my ($self, $sid, $cd, $sd, $lrm_res) = @_;
 390
 391     my $haenv = $self->{haenv};
 392     my $ns = $self->{ns};
 393
 394     # check result from LRM daemon
 395     if ($lrm_res) {
 396         my $exit_code = $lrm_res->{exit_code};
 397         if ($exit_code == 0) {
 398             &$change_service_state($self, $sid, 'started', node => $sd->{target});
 399             return;
 400         } else {
 401             $haenv->log('err', "service '$sid' - migration failed (exit code $exit_code)");
 402             &$change_service_state($self, $sid, 'started', node => $sd->{node});
 403             return;
 404         }
 405     }
 406
 407     if (!$ns->node_is_online($sd->{node})) {
 408         &$change_service_state($self, $sid, 'fence');
 409         return;
 410     }
 411 }
 412
 413
 414 sub next_state_stopped {
 415     my ($self, $sid, $cd, $sd, $lrm_res) = @_;
 416
 417     my $haenv = $self->{haenv};
 418     my $ns = $self->{ns};
 419
 420     if ($sd->{node} ne $cd->{node}) {
 421         # this can happen if we fence a node with active migrations
 422         # hack: modify $sd (normally this should be considered read-only)
 423         $haenv->log('info', "fixup service '$sid' location ($sd->{node} => $cd->{node}");
 424         $sd->{node} = $cd->{node};
 425     }
 426
 427     if ($sd->{cmd}) {
 428         my ($cmd, $target) = @{$sd->{cmd}};
 429         delete $sd->{cmd};
 430
 431         if ($cmd eq 'migrate' || $cmd eq 'relocate') {
 432             if (!$ns->node_is_online($target)) {
 433                 $haenv->log('err', "ignore service '$sid' $cmd request - node '$target' not online");
 434             } elsif ($sd->{node} eq $target) {
 435                 $haenv->log('info', "ignore service '$sid' $cmd request - service already on node '$target'");
 436             } else {
 437                 $haenv->change_service_location($sid, $target);
 438                 $cd->{node} = $sd->{node} = $target; # fixme: $sd is read-only??!!
 439                 $haenv->log('info', "$cmd service '$sid' to node '$target' (stopped)");
 440             }
 441         } else {
 442             $haenv->log('err', "unknown command '$cmd' for service '$sid'");
 443         }
 444     }
 445
 446     if ($cd->{state} eq 'disabled') {
 447         # do nothing
 448         return;
 449     }
 450
 451     if ($cd->{state} eq 'enabled') {
 452         if (my $node = select_service_node($self->{groups}, $self->{online_node_usage}, $cd, $sd->{node})) {
 453             if ($node && ($sd->{node} ne $node)) {
 454                 $haenv->change_service_location($sid, $node);
 455             }
 456             &$change_service_state($self, $sid, 'started', node => $node);
 457         } else {
 458             # fixme: warn
 459         }
 460
 461         return;
 462     }
 463
 464     $haenv->log('err', "service '$sid' - unknown state '$cd->{state}' in service configuration");
 465 }
 466
 467 sub next_state_started {
 468     my ($self, $sid, $cd, $sd, $lrm_res) = @_;
 469
 470     my $haenv = $self->{haenv};
 471     my $ns = $self->{ns};
 472
 473     if (!$ns->node_is_online($sd->{node})) {
 474
 475         &$change_service_state($self, $sid, 'fence');
 476         return;
 477     }
 478
 479     if ($cd->{state} eq 'disabled') {
 480         &$change_service_state($self, $sid, 'request_stop');
 481         return;
 482     }
 483
 484     if ($cd->{state} eq 'enabled') {
 485
 486         if ($sd->{cmd}) {
 487             my ($cmd, $target) = @{$sd->{cmd}};
 488             delete $sd->{cmd};
 489
 490             if ($cmd eq 'migrate' || $cmd eq 'relocate') {
 491                 if (!$ns->node_is_online($target)) {
 492                     $haenv->log('err', "ignore service '$sid' $cmd request - node '$target' not online");
 493                 } elsif ($sd->{node} eq $target) {
 494                     $haenv->log('info', "ignore service '$sid' $cmd request - service already on node '$target'");
 495                 } else {
 496                     $haenv->log('info', "$cmd service '$sid' to node '$target' (running)");
 497                     &$change_service_state($self, $sid, $cmd, node => $sd->{node}, target => $target);
 498                 }
 499             } else {
 500                 $haenv->log('err', "unknown command '$cmd' for service '$sid'");
 501             }
 502         } else {
 503
 504             my $try_next = 0;
 505             if ($lrm_res && ($lrm_res->{exit_code} != 0)) { # fixme: other exit codes?
 506                 $try_next = 1;
 507             }
 508
 509             my $node = select_service_node($self->{groups}, $self->{online_node_usage},
 510                                            $cd, $sd->{node}, $try_next);
 511
 512             if ($node && ($sd->{node} ne $node)) {
 513                 $haenv->log('info', "migrate service '$sid' to node '$node' (running)");
 514                 &$change_service_state($self, $sid, 'migrate', node => $sd->{node}, target => $node);
 515             } else {
 516                 # do nothing
 517             }
 518         }
 519
 520         return;
 521     }
 522
 523     $haenv->log('err', "service '$sid' - unknown state '$cd->{state}' in service configuration");
 524 }
 525
 526 1;