src/PVE/HA/Manager.pm

   1 package PVE::HA::Manager;
   2
   3 use strict;
   4 use warnings;
   5 use Digest::MD5 qw(md5_base64);
   6
   7 use Data::Dumper;
   8 use PVE::Tools;
   9 use PVE::HA::Tools ':exit_codes';
  10 use PVE::HA::NodeStatus;
  11
  12 my $fence_delay = 60;
  13
  14 sub new {
  15     my ($this, $haenv) = @_;
  16
  17     my $class = ref($this) || $this;
  18
  19     my $ms = $haenv->read_manager_status();
  20
  21     $ms->{master_node} = $haenv->nodename();
  22
  23     my $ns = PVE::HA::NodeStatus->new($haenv, $ms->{node_status} || {});
  24
  25     # fixme: use separate class  PVE::HA::ServiceStatus
  26     my $ss = $ms->{service_status} || {};
  27
  28     my $self = bless {
  29         haenv => $haenv,
  30         ms => $ms, # master status
  31         ns => $ns, # PVE::HA::NodeStatus
  32         ss => $ss, # service status
  33     }, $class;
  34
  35     return $self;
  36 }
  37
  38 sub cleanup {
  39     my ($self) = @_;
  40
  41     # todo: ?
  42 }
  43
  44 sub flush_master_status {
  45     my ($self) = @_;
  46
  47     my ($haenv, $ms, $ns, $ss) = ($self->{haenv}, $self->{ms}, $self->{ns}, $self->{ss});
  48
  49     $ms->{node_status} = $ns->{status};
  50     $ms->{service_status} = $ss;
  51     $ms->{timestamp} = $haenv->get_time();
  52
  53     $haenv->write_manager_status($ms);
  54 }
  55
  56 sub select_service_node {
  57     my ($groups, $online_node_usage, $service_conf, $current_node, $try_next) = @_;
  58
  59     my $group = { 'nodes' => { $service_conf->{node} => 1 } }; # default group
  60
  61     $group =  $groups->{ids}->{$service_conf->{group}} if $service_conf->{group} &&
  62         $groups->{ids}->{$service_conf->{group}};
  63
  64     my $pri_groups = {};
  65     my $group_members = {};
  66     foreach my $entry (keys %{$group->{nodes}}) {
  67         my ($node, $pri) = ($entry, 0);
  68         if ($entry =~ m/^(\S+):(\d+)$/) {
  69             ($node, $pri) = ($1, $2);
  70         }
  71         next if !defined($online_node_usage->{$node}); # offline
  72         $pri_groups->{$pri}->{$node} = 1;
  73         $group_members->{$node} = $pri;
  74     }
  75
  76
  77     # add non-group members to unrestricted groups (priority -1)
  78     if (!$group->{restricted}) {
  79         my $pri = -1;
  80         foreach my $node (keys %$online_node_usage) {
  81             next if defined($group_members->{$node});
  82             $pri_groups->{$pri}->{$node} = 1;
  83             $group_members->{$node} = -1;
  84         }
  85     }
  86
  87
  88     my @pri_list = sort {$b <=> $a} keys %$pri_groups;
  89     return undef if !scalar(@pri_list);
  90
  91     if (!$try_next && $group->{nofailback} && defined($group_members->{$current_node})) {
  92         return $current_node;
  93     }
  94
  95     # select node from top priority node list
  96
  97     my $top_pri = $pri_list[0];
  98
  99     my @nodes = sort {
 100         $online_node_usage->{$a} <=> $online_node_usage->{$b} || $a cmp $b
 101     } keys %{$pri_groups->{$top_pri}};
 102
 103     my $found;
 104     for (my $i = scalar(@nodes) - 1; $i >= 0; $i--) {
 105         my $node = $nodes[$i];
 106         if ($node eq $current_node) {
 107             $found = $i;
 108             last;
 109         }
 110     }
 111
 112     if ($try_next) {
 113
 114         if (defined($found) && ($found < (scalar(@nodes) - 1))) {
 115             return $nodes[$found + 1];
 116         } else {
 117             return $nodes[0];
 118         }
 119
 120     } else {
 121
 122         return $nodes[$found] if defined($found);
 123
 124         return $nodes[0];
 125
 126     }
 127 }
 128
 129 my $uid_counter = 0;
 130
 131 sub compute_new_uuid {
 132     my ($state) = @_;
 133
 134     $uid_counter++;
 135     return md5_base64($state . $$ . time() . $uid_counter);
 136 }
 137
 138 my $valid_service_states = {
 139     stopped => 1,
 140     request_stop => 1,
 141     started => 1,
 142     fence => 1,
 143     migrate => 1,
 144     relocate => 1,
 145     freeze => 1,
 146     error => 1,
 147 };
 148
 149 sub recompute_online_node_usage {
 150     my ($self) = @_;
 151
 152     my $online_node_usage = {};
 153
 154     my $online_nodes = $self->{ns}->list_online_nodes();
 155
 156     foreach my $node (@$online_nodes) {
 157         $online_node_usage->{$node} = 0;
 158     }
 159
 160     foreach my $sid (keys %{$self->{ss}}) {
 161         my $sd = $self->{ss}->{$sid};
 162         my $state = $sd->{state};
 163         if (defined($online_node_usage->{$sd->{node}})) {
 164             if (($state eq 'started') || ($state eq 'request_stop') ||
 165                 ($state eq 'fence') || ($state eq 'freeze') || ($state eq 'error')) {
 166                 $online_node_usage->{$sd->{node}}++;
 167             } elsif (($state eq 'migrate') || ($state eq 'relocate')) {
 168                 $online_node_usage->{$sd->{target}}++;
 169             } elsif ($state eq 'stopped') {
 170                 # do nothing
 171             } else {
 172                 die "should not be reached";
 173             }
 174         }
 175     }
 176
 177     $self->{online_node_usage} = $online_node_usage;
 178 }
 179
 180 my $change_service_state = sub {
 181     my ($self, $sid, $new_state, %params) = @_;
 182
 183     my ($haenv, $ss) = ($self->{haenv}, $self->{ss});
 184
 185     my $sd = $ss->{$sid} || die "no such service '$sid";
 186
 187     my $old_state = $sd->{state};
 188     my $old_node = $sd->{node};
 189
 190     die "no state change" if $old_state eq $new_state; # just to be sure
 191
 192     die "invalid CRM service state '$new_state'\n" if !$valid_service_states->{$new_state};
 193
 194     foreach my $k (keys %$sd) { delete $sd->{$k}; };
 195
 196     $sd->{state} = $new_state;
 197     $sd->{node} = $old_node;
 198
 199     my $text_state = '';
 200     foreach my $k (sort keys %params) {
 201         my $v = $params{$k};
 202         $text_state .= ", " if $text_state;
 203         $text_state .= "$k = $v";
 204         $sd->{$k} = $v;
 205     }
 206
 207     $self->recompute_online_node_usage();
 208
 209     $sd->{uid} = compute_new_uuid($new_state);
 210
 211
 212     $text_state = " ($text_state)" if $text_state;
 213     $haenv->log('info', "service '$sid': state changed from '${old_state}' to '${new_state}' $text_state");
 214 };
 215
 216 # read LRM status for all nodes
 217 sub read_lrm_status {
 218     my ($self) = @_;
 219
 220     my $nodes = $self->{ns}->list_nodes();
 221     my $haenv = $self->{haenv};
 222
 223     my $results = {};
 224     my $modes = {};
 225     foreach my $node (@$nodes) {
 226         my $lrm_status = $haenv->read_lrm_status($node);
 227         $modes->{$node} = $lrm_status->{mode} || 'active';
 228         foreach my $uid (keys %{$lrm_status->{results}}) {
 229             next if $results->{$uid}; # should not happen
 230             $results->{$uid} = $lrm_status->{results}->{$uid};
 231         }
 232     }
 233
 234
 235     return ($results, $modes);
 236 }
 237
 238 # read new crm commands and save them into crm master status
 239 sub update_crm_commands {
 240     my ($self) = @_;
 241
 242     my ($haenv, $ms, $ns, $ss) = ($self->{haenv}, $self->{ms}, $self->{ns}, $self->{ss});
 243
 244     my $cmdlist = $haenv->read_crm_commands();
 245
 246     foreach my $cmd (split(/\n/, $cmdlist)) {
 247         chomp $cmd;
 248
 249         if ($cmd =~ m/^(migrate|relocate)\s+(\S+)\s+(\S+)$/) {
 250             my ($task, $sid, $node) = ($1, $2, $3);
 251             if (my $sd = $ss->{$sid}) {
 252                 if (!$ns->node_is_online($node)) {
 253                     $haenv->log('err', "crm command error - node not online: $cmd");
 254                 } else {
 255                     if ($node eq $sd->{node}) {
 256                         $haenv->log('info', "ignore crm command - service already on target node: $cmd");
 257                     } else {
 258                         $haenv->log('info', "got crm command: $cmd");
 259                         $ss->{$sid}->{cmd} = [ $task, $node];
 260                     }
 261                 }
 262             } else {
 263                 $haenv->log('err', "crm command error - no such service: $cmd");
 264             }
 265
 266         } else {
 267             $haenv->log('err', "unable to parse crm command: $cmd");
 268         }
 269     }
 270
 271 }
 272
 273 sub manage {
 274     my ($self) = @_;
 275
 276     my ($haenv, $ms, $ns, $ss) = ($self->{haenv}, $self->{ms}, $self->{ns}, $self->{ss});
 277
 278     $ns->update($haenv->get_node_info());
 279
 280     if (!$ns->node_is_online($haenv->nodename())) {
 281         $haenv->log('info', "master seems offline");
 282         return;
 283     }
 284
 285     my ($lrm_results, $lrm_modes) = $self->read_lrm_status();
 286
 287     my $sc = $haenv->read_service_config();
 288
 289     $self->{groups} = $haenv->read_group_config(); # update
 290
 291     # compute new service status
 292
 293     # add new service
 294     foreach my $sid (sort keys %$sc) {
 295         next if $ss->{$sid}; # already there
 296         $haenv->log('info', "adding new service '$sid' on node '$sc->{$sid}->{node}'");
 297         # assume we are running to avoid relocate running service at add
 298         $ss->{$sid} = { state => 'started', node => $sc->{$sid}->{node},
 299                         uid => compute_new_uuid('started') };
 300     }
 301
 302     # remove stale service from manager state
 303     foreach my $sid (keys %$ss) {
 304         next if $sc->{$sid};
 305         $haenv->log('info', "removing stale service '$sid' (no config)");
 306         delete $ss->{$sid};
 307     }
 308
 309     $self->update_crm_commands();
 310
 311     for (;;) {
 312         my $repeat = 0;
 313
 314         $self->recompute_online_node_usage();
 315
 316         foreach my $sid (keys %$ss) {
 317             my $sd = $ss->{$sid};
 318             my $cd = $sc->{$sid} || { state => 'disabled' };
 319
 320             my $lrm_res = $sd->{uid} ? $lrm_results->{$sd->{uid}} : undef;
 321
 322             my $last_state = $sd->{state};
 323
 324             if ($last_state eq 'stopped') {
 325
 326                 $self->next_state_stopped($sid, $cd, $sd, $lrm_res);
 327
 328             } elsif ($last_state eq 'started') {
 329
 330                 $self->next_state_started($sid, $cd, $sd, $lrm_res);
 331
 332             } elsif ($last_state eq 'migrate' || $last_state eq 'relocate') {
 333
 334                 $self->next_state_migrate_relocate($sid, $cd, $sd, $lrm_res);
 335
 336             } elsif ($last_state eq 'fence') {
 337
 338                 # do nothing here - wait until fenced
 339
 340             } elsif ($last_state eq 'request_stop') {
 341
 342                 $self->next_state_request_stop($sid, $cd, $sd, $lrm_res);
 343
 344             } elsif ($last_state eq 'freeze') {
 345
 346                 my $lrm_mode = $sd->{node} ? $lrm_modes->{$sd->{node}} : undef;
 347                 # unfreeze
 348                 &$change_service_state($self, $sid, 'started')
 349                     if $lrm_mode && $lrm_mode eq 'active';
 350
 351             } elsif ($last_state eq 'error') {
 352
 353                 $self->next_state_error($sid, $cd, $sd, $lrm_res);
 354
 355             } else {
 356
 357                 die "unknown service state '$last_state'";
 358             }
 359
 360             my $lrm_mode = $sd->{node} ? $lrm_modes->{$sd->{node}} : undef;
 361             if ($lrm_mode && $lrm_mode eq 'restart') {
 362                 if (($sd->{state} eq 'started' || $sd->{state} eq 'stopped' ||
 363                      $sd->{state} eq 'request_stop')) {
 364                     &$change_service_state($self, $sid, 'freeze');
 365                 }
 366             }
 367
 368             $repeat = 1 if $sd->{state} ne $last_state;
 369         }
 370
 371         # handle fencing
 372         my $fenced_nodes = {};
 373         foreach my $sid (keys %$ss) {
 374             my $sd = $ss->{$sid};
 375             next if $sd->{state} ne 'fence';
 376
 377             if (!defined($fenced_nodes->{$sd->{node}})) {
 378                 $fenced_nodes->{$sd->{node}} = $ns->fence_node($sd->{node}) || 0;
 379             }
 380
 381             next if !$fenced_nodes->{$sd->{node}};
 382
 383             # node fence was successful - mark service as stopped
 384             &$change_service_state($self, $sid, 'stopped');
 385         }
 386
 387         last if !$repeat;
 388     }
 389
 390     $self->flush_master_status();
 391 }
 392
 393 # functions to compute next service states
 394 # $cd: service configuration data (read only)
 395 # $sd: service status data (read only)
 396 #
 397 # Note: use change_service_state() to alter state
 398 #
 399
 400 sub next_state_request_stop {
 401     my ($self, $sid, $cd, $sd, $lrm_res) = @_;
 402
 403     my $haenv = $self->{haenv};
 404     my $ns = $self->{ns};
 405
 406     # check result from LRM daemon
 407     if ($lrm_res) {
 408         my $exit_code = $lrm_res->{exit_code};
 409         if ($exit_code == SUCCESS) {
 410             &$change_service_state($self, $sid, 'stopped');
 411             return;
 412         } else {
 413             $haenv->log('err', "service '$sid' stop failed (exit code $exit_code)");
 414             &$change_service_state($self, $sid, 'error'); # fixme: what state?
 415             return;
 416         }
 417     }
 418
 419     if ($ns->node_is_offline_delayed($sd->{node}, $fence_delay)) {
 420         &$change_service_state($self, $sid, 'fence');
 421         return;
 422     }
 423 }
 424
 425 sub next_state_migrate_relocate {
 426     my ($self, $sid, $cd, $sd, $lrm_res) = @_;
 427
 428     my $haenv = $self->{haenv};
 429     my $ns = $self->{ns};
 430
 431     # check result from LRM daemon
 432     if ($lrm_res) {
 433         my $exit_code = $lrm_res->{exit_code};
 434         if ($exit_code == SUCCESS) {
 435             &$change_service_state($self, $sid, 'started', node => $sd->{target});
 436             return;
 437         } else {
 438             $haenv->log('err', "service '$sid' - migration failed (exit code $exit_code)");
 439             &$change_service_state($self, $sid, 'started', node => $sd->{node});
 440             return;
 441         }
 442     }
 443
 444     if ($ns->node_is_offline_delayed($sd->{node}, $fence_delay)) {
 445         &$change_service_state($self, $sid, 'fence');
 446         return;
 447     }
 448 }
 449
 450
 451 sub next_state_stopped {
 452     my ($self, $sid, $cd, $sd, $lrm_res) = @_;
 453
 454     my $haenv = $self->{haenv};
 455     my $ns = $self->{ns};
 456
 457     if ($sd->{node} ne $cd->{node}) {
 458         # this can happen if we fence a node with active migrations
 459         # hack: modify $sd (normally this should be considered read-only)
 460         $haenv->log('info', "fixup service '$sid' location ($sd->{node} => $cd->{node}");
 461         $sd->{node} = $cd->{node};
 462     }
 463
 464     if ($sd->{cmd}) {
 465         my ($cmd, $target) = @{$sd->{cmd}};
 466         delete $sd->{cmd};
 467
 468         if ($cmd eq 'migrate' || $cmd eq 'relocate') {
 469             if (!$ns->node_is_online($target)) {
 470                 $haenv->log('err', "ignore service '$sid' $cmd request - node '$target' not online");
 471             } elsif ($sd->{node} eq $target) {
 472                 $haenv->log('info', "ignore service '$sid' $cmd request - service already on node '$target'");
 473             } else {
 474                 eval {
 475                     $haenv->change_service_location($sid, $sd->{node}, $target);
 476                     $cd->{node} = $sd->{node} = $target; # fixme: $sd is read-only??!!
 477                     $haenv->log('info', "$cmd service '$sid' to node '$target' (stopped)");
 478                 };
 479                 if (my $err = $@) {
 480                     $haenv->log('err', "$cmd service '$sid' to node '$target' failed - $err");
 481                 }
 482             }
 483         } else {
 484             $haenv->log('err', "unknown command '$cmd' for service '$sid'");
 485         }
 486     }
 487
 488     if ($cd->{state} eq 'disabled') {
 489         # do nothing
 490         return;
 491     }
 492
 493     if ($cd->{state} eq 'enabled') {
 494         if (my $node = select_service_node($self->{groups}, $self->{online_node_usage}, $cd, $sd->{node})) {
 495             if ($node && ($sd->{node} ne $node)) {
 496                 eval {
 497                     $haenv->change_service_location($sid, $sd->{node}, $node);
 498                     $cd->{node} = $sd->{node} = $node; # fixme: $sd is read-only??!!
 499                 };
 500                 if (my $err = $@) {
 501                     $haenv->log('err', "move service '$sid' to node '$node' failed - $err");
 502                 } else {
 503                     &$change_service_state($self, $sid, 'started', node => $node);
 504                 }
 505             } else {
 506                 &$change_service_state($self, $sid, 'started', node => $node);
 507             }
 508         } else {
 509             # fixme: warn
 510         }
 511
 512         return;
 513     }
 514
 515     $haenv->log('err', "service '$sid' - unknown state '$cd->{state}' in service configuration");
 516 }
 517
 518 sub next_state_started {
 519     my ($self, $sid, $cd, $sd, $lrm_res) = @_;
 520
 521     my $haenv = $self->{haenv};
 522     my $master_status = $self->{ms};
 523     my $ns = $self->{ns};
 524
 525     if (!$ns->node_is_online($sd->{node})) {
 526         if ($ns->node_is_offline_delayed($sd->{node}, $fence_delay)) {
 527             &$change_service_state($self, $sid, 'fence');
 528         }
 529         return;
 530     }
 531
 532     if ($cd->{state} eq 'disabled') {
 533         &$change_service_state($self, $sid, 'request_stop');
 534         return;
 535     }
 536
 537     if ($cd->{state} eq 'enabled') {
 538
 539         if ($sd->{cmd}) {
 540             my ($cmd, $target) = @{$sd->{cmd}};
 541             delete $sd->{cmd};
 542
 543             if ($cmd eq 'migrate' || $cmd eq 'relocate') {
 544                 if (!$ns->node_is_online($target)) {
 545                     $haenv->log('err', "ignore service '$sid' $cmd request - node '$target' not online");
 546                 } elsif ($sd->{node} eq $target) {
 547                     $haenv->log('info', "ignore service '$sid' $cmd request - service already on node '$target'");
 548                 } else {
 549                     $haenv->log('info', "$cmd service '$sid' to node '$target' (running)");
 550                     &$change_service_state($self, $sid, $cmd, node => $sd->{node}, target => $target);
 551                 }
 552             } else {
 553                 $haenv->log('err', "unknown command '$cmd' for service '$sid'");
 554             }
 555         } else {
 556
 557             my $try_next = 0;
 558             if ($lrm_res) {
 559                 if ($lrm_res->{exit_code} == ERROR) {
 560
 561                     my $try = $master_status->{relocate_trial}->{$sid} || 0;
 562
 563                     if ($try < $cd->{max_relocate}) {
 564
 565                         $try++;
 566                         $try_next = 1; # tell select_service_node to relocate
 567
 568                         $haenv->log('warning', "starting service $sid on node".
 569                                    " '$sd->{node}' failed, relocating service.");
 570                         $master_status->{relocate_trial}->{$sid} = $try;
 571
 572                     } else {
 573
 574                         $haenv->log('err', "recovery policy for service".
 575                                    " $sid failed, entering error state!");
 576                         &$change_service_state($self, $sid, 'error');
 577                         return;
 578
 579                     }
 580                 } elsif ($lrm_res->{exit_code} == SUCCESS) {
 581                     $master_status->{relocate_trial}->{$sid} = 0;
 582                 }
 583             }
 584
 585             my $node = select_service_node($self->{groups}, $self->{online_node_usage},
 586                                            $cd, $sd->{node}, $try_next);
 587
 588             if ($node && ($sd->{node} ne $node)) {
 589                 if ($cd->{type} eq 'vm') {
 590                     $haenv->log('info', "migrate service '$sid' to node '$node' (running)");
 591                     &$change_service_state($self, $sid, 'migrate', node => $sd->{node}, target => $node);
 592                 } else {
 593                     $haenv->log('info', "relocate service '$sid' to node '$node'");
 594                     &$change_service_state($self, $sid, 'relocate', node => $sd->{node}, target => $node);
 595                 }
 596             } else {
 597                 # do nothing
 598             }
 599         }
 600
 601         return;
 602     }
 603
 604     $haenv->log('err', "service '$sid' - unknown state '$cd->{state}' in service configuration");
 605 }
 606
 607 sub next_state_error {
 608     my ($self, $sid, $cd, $sd, $lrm_res) = @_;
 609
 610     my $ns = $self->{ns};
 611
 612     if ($cd->{state} eq 'disabled') {
 613         &$change_service_state($self, $sid, 'stopped');
 614         return;
 615     }
 616
 617     if ($ns->node_is_offline_delayed($sd->{node}, $fence_delay)) {
 618         &$change_service_state($self, $sid, 'fence');
 619         return;
 620     }
 621
 622 }
 623
 624 1;