X-Git-Url: https://git.proxmox.com/?a=blobdiff_plain;f=src%2FPVE%2FHA%2FManager.pm;h=9e46f19d6ff55ef1c2721bd5a8790a1a450c528d;hb=2167dd1e6093362ccfbd3ad55c4c4bd0dcebc73d;hp=bbcf5e7c79ba541964203fb72f2dc8578ecadb63;hpb=9adedcd738df0e861466b3879d6e86a1bc65ebb3;p=pve-ha-manager.git

diff --git a/src/PVE/HA/Manager.pm b/src/PVE/HA/Manager.pm
index bbcf5e7..9e46f19 100644
--- a/src/PVE/HA/Manager.pm
+++ b/src/PVE/HA/Manager.pm
@@ -43,9 +43,9 @@ sub flush_master_status {
     $ms->{node_status} = $ns->{status};
     $ms->{service_status} = $ss;
     $ms->{timestamp} = $haenv->get_time();
-    
+
     $haenv->write_manager_status($ms);
-} 
+}
 
 sub get_service_group {
     my ($groups, $online_node_usage, $service_conf) = @_;
@@ -57,7 +57,7 @@ sub get_service_group {
     }
 
     # overwrite default if service is bound to a specific group
-    $group =  $groups->{ids}->{$service_conf->{group}} if $service_conf->{group} && 
+    $group =  $groups->{ids}->{$service_conf->{group}} if $service_conf->{group} &&
 	$groups->{ids}->{$service_conf->{group}};
 
     return $group;
@@ -93,7 +93,7 @@ sub get_node_priority_groups {
 }
 
 sub select_service_node {
-    my ($groups, $online_node_usage, $service_conf, $current_node, $try_next, $tried_nodes) = @_;
+    my ($groups, $online_node_usage, $service_conf, $current_node, $try_next, $tried_nodes, $maintenance_fallback) = @_;
 
     my $group = get_service_group($groups, $online_node_usage, $service_conf);
 
@@ -118,33 +118,36 @@ sub select_service_node {
 	}
     }
 
-    my @nodes = sort { 
+    my @nodes = sort {
 	$online_node_usage->{$a} <=> $online_node_usage->{$b} || $a cmp $b
     } keys %{$pri_groups->{$top_pri}};
 
     my $found;
+    my $found_maintenace_fallback;
     for (my $i = scalar(@nodes) - 1; $i >= 0; $i--) {
 	my $node = $nodes[$i];
 	if ($node eq $current_node) {
 	    $found = $i;
-	    last;
+	}
+	if (defined($maintenance_fallback) && $node eq $maintenance_fallback) {
+	    $found_maintenace_fallback = $i;
 	}
     }
 
-    if ($try_next) {
+    if (defined($found_maintenace_fallback)) {
+	return $nodes[$found_maintenace_fallback];
+    }
 
+    if ($try_next) {
 	if (defined($found) && ($found < (scalar(@nodes) - 1))) {
 	    return $nodes[$found + 1];
 	} else {
 	    return $nodes[0];
 	}
-
+    } elsif (defined($found)) {
+	return $nodes[$found];
     } else {
-
-	return $nodes[$found] if defined($found);
-
 	return $nodes[0];
-
     }
 }
 
@@ -152,7 +155,7 @@ my $uid_counter = 0;
 
 sub compute_new_uuid {
     my ($state) = @_;
-    
+
     $uid_counter++;
     return md5_base64($state . $$ . time() . $uid_counter);
 }
@@ -183,10 +186,12 @@ sub recompute_online_node_usage {
 	my $sd = $self->{ss}->{$sid};
 	my $state = $sd->{state};
 	if (defined($online_node_usage->{$sd->{node}})) {
-	    if (($state eq 'started') || ($state eq 'request_stop') || 
+	    if (($state eq 'started') || ($state eq 'request_stop') ||
 		($state eq 'fence') || ($state eq 'freeze') || ($state eq 'error')) {
 		$online_node_usage->{$sd->{node}}++;
 	    } elsif (($state eq 'migrate') || ($state eq 'relocate')) {
+		# count it for both, source and target as load is put on both
+		$online_node_usage->{$sd->{node}}++;
 		$online_node_usage->{$sd->{target}}++;
 	    } elsif ($state eq 'stopped') {
 		# do nothing
@@ -209,6 +214,7 @@ my $change_service_state = sub {
     my $old_state = $sd->{state};
     my $old_node = $sd->{node};
     my $old_failed_nodes = $sd->{failed_nodes};
+    my $old_maintenance_node = $sd->{maintenance_node};
 
     die "no state change" if $old_state eq $new_state; # just to be sure
 
@@ -218,7 +224,8 @@ my $change_service_state = sub {
 
     $sd->{state} = $new_state;
     $sd->{node} = $old_node;
-    $sd->{failed_nodes} = $old_failed_nodes;
+    $sd->{failed_nodes} = $old_failed_nodes if defined($old_failed_nodes);
+    $sd->{maintenance_node} = $old_maintenance_node if defined($old_maintenance_node);
 
     my $text_state = '';
     foreach my $k (sort keys %params) {
@@ -243,14 +250,15 @@ my $fence_recovery_cleanup = sub {
 
     my $haenv = $self->{haenv};
 
-    my (undef, $type, $id) = PVE::HA::Tools::parse_sid($sid);
+    my (undef, $type, $id) = $haenv->parse_sid($sid);
     my $plugin = PVE::HA::Resources->lookup($type);
 
     # should not happen
     die "unknown resource type '$type'" if !$plugin;
 
-    # locks may block recovery, cleanup those which are safe to remove after fencing
-    my $removable_locks = ['backup', 'mounted'];
+    # locks may block recovery, cleanup those which are safe to remove after fencing,
+    # i.e., after the original node was reset and thus all it's state
+    my $removable_locks = ['backup', 'mounted', 'migrate', 'clone', 'rollback', 'snapshot', 'snapshot-delete', 'suspending', 'suspended'];
     if (my $removed_lock = $plugin->remove_locks($haenv, $id, $removable_locks, $fenced_node)) {
 	$haenv->log('warning', "removed leftover lock '$removed_lock' from recovered " .
 	            "service '$sid' to allow its start.");
@@ -286,6 +294,7 @@ my $recover_fenced_service = sub {
 	&$fence_recovery_cleanup($self, $sid, $fenced_node);
 
 	$haenv->steal_service($sid, $sd->{node}, $recovery_node);
+	$self->{online_node_usage}->{$recovery_node}++;
 
 	# $sd *is normally read-only*, fencing is the exception
 	$cd->{node} = $sd->{node} = $recovery_node;
@@ -299,7 +308,7 @@ my $recover_fenced_service = sub {
     }
 };
 
-# read LRM status for all nodes 
+# read LRM status for all nodes
 sub read_lrm_status {
     my ($self) = @_;
 
@@ -317,7 +326,7 @@ sub read_lrm_status {
 	}
     }
 
-    
+
     return ($results, $modes);
 }
 
@@ -328,27 +337,35 @@ sub update_crm_commands {
     my ($haenv, $ms, $ns, $ss) = ($self->{haenv}, $self->{ms}, $self->{ns}, $self->{ss});
 
     my $cmdlist = $haenv->read_crm_commands();
-    
+
     foreach my $cmd (split(/\n/, $cmdlist)) {
 	chomp $cmd;
 
 	if ($cmd =~ m/^(migrate|relocate)\s+(\S+)\s+(\S+)$/) {
-	    my ($task, $sid, $node) = ($1, $2, $3); 
+	    my ($task, $sid, $node) = ($1, $2, $3);
 	    if (my $sd = $ss->{$sid}) {
 		if (!$ns->node_is_online($node)) {
 		    $haenv->log('err', "crm command error - node not online: $cmd");
 		} else {
 		    if ($node eq $sd->{node}) {
 			$haenv->log('info', "ignore crm command - service already on target node: $cmd");
-		    } else { 
+		    } else {
 			$haenv->log('info', "got crm command: $cmd");
-			$ss->{$sid}->{cmd} = [ $task, $node];
+			$ss->{$sid}->{cmd} = [ $task, $node ];
 		    }
 		}
 	    } else {
 		$haenv->log('err', "crm command error - no such service: $cmd");
 	    }
 
+	} elsif ($cmd =~ m/^stop\s+(\S+)\s+(\S+)$/) {
+	    my ($sid, $timeout) = ($1, $2);
+	    if (my $sd = $ss->{$sid}) {
+		$haenv->log('info', "got crm command: $cmd");
+		$ss->{$sid}->{cmd} = [ 'stop', $timeout ];
+	    } else {
+		$haenv->log('err', "crm command error - no such service: $cmd");
+	    }
 	} else {
 	    $haenv->log('err', "unable to parse crm command: $cmd");
 	}
@@ -361,15 +378,16 @@ sub manage {
 
     my ($haenv, $ms, $ns, $ss) = ($self->{haenv}, $self->{ms}, $self->{ns}, $self->{ss});
 
-    $ns->update($haenv->get_node_info());
+    my ($node_info) = $haenv->get_node_info();
+    my ($lrm_results, $lrm_modes) = $self->read_lrm_status();
 
-    if (!$ns->node_is_online($haenv->nodename())) {
+    $ns->update($node_info, $lrm_modes);
+
+    if (!$ns->node_is_operational($haenv->nodename())) {
 	$haenv->log('info', "master seems offline");
 	return;
     }
 
-    my ($lrm_results, $lrm_modes) = $self->read_lrm_status();
-
     my $sc = $haenv->read_service_config();
 
     $self->{groups} = $haenv->read_group_config(); # update
@@ -380,6 +398,8 @@ sub manage {
     foreach my $sid (sort keys %$sc) {
 	next if $ss->{$sid}; # already there
 	my $cd = $sc->{$sid};
+	next if $cd->{state} eq 'ignored';
+
 	$haenv->log('info', "adding new service '$sid' on node '$cd->{node}'");
 	# assume we are running to avoid relocate running service at add
 	my $state = ($cd->{state} eq 'started') ? 'started' : 'request_stop';
@@ -387,10 +407,13 @@ sub manage {
 			uid => compute_new_uuid('started') };
     }
 
-    # remove stale service from manager state
+    # remove stale or ignored services from manager state
     foreach my $sid (keys %$ss) {
-	next if $sc->{$sid};
-	$haenv->log('info', "removing stale service '$sid' (no config)");
+	next if $sc->{$sid} && $sc->{$sid}->{state} ne 'ignored';
+
+	my $reason =  defined($sc->{$sid}) ? 'ignored state requested' : 'no config';
+	$haenv->log('info', "removing stale service '$sid' ($reason)");
+
 	# remove all service related state information
 	delete $ss->{$sid};
     }
@@ -399,7 +422,7 @@ sub manage {
 
     for (;;) {
 	my $repeat = 0;
-	
+
 	$self->recompute_online_node_usage();
 
 	foreach my $sid (sort keys %$ss) {
@@ -542,7 +565,6 @@ sub next_state_migrate_relocate {
     }
 }
 
-
 sub next_state_stopped {
     my ($self, $sid, $cd, $sd, $lrm_res) = @_;
 
@@ -553,14 +575,14 @@ sub next_state_stopped {
 	# this can happen if we fence a node with active migrations
 	# hack: modify $sd (normally this should be considered read-only)
 	$haenv->log('info', "fixup service '$sid' location ($sd->{node} => $cd->{node})");
-	$sd->{node} = $cd->{node}; 
+	$sd->{node} = $cd->{node};
     }
 
     if ($sd->{cmd}) {
-	my ($cmd, $target) = @{$sd->{cmd}};
-	delete $sd->{cmd};
+	my $cmd = shift @{$sd->{cmd}};
 
 	if ($cmd eq 'migrate' || $cmd eq 'relocate') {
+	    my $target = shift @{$sd->{cmd}};
 	    if (!$ns->node_is_online($target)) {
 		$haenv->log('err', "ignore service '$sid' $cmd request - node '$target' not online");
 	    } elsif ($sd->{node} eq $target) {
@@ -570,9 +592,12 @@ sub next_state_stopped {
 				       target => $target);
 		return;
 	    }
+	} elsif ($cmd eq 'stop') {
+		$haenv->log('info', "ignore service '$sid' $cmd request - service already stopped");
 	} else {
-	    $haenv->log('err', "unknown command '$cmd' for service '$sid'"); 
+	    $haenv->log('err', "unknown command '$cmd' for service '$sid'");
 	}
+	delete $sd->{cmd};
     }
 
     if ($cd->{state} eq 'disabled') {
@@ -623,9 +648,15 @@ sub next_state_started {
 	if ($ns->node_is_offline_delayed($sd->{node})) {
 	    &$change_service_state($self, $sid, 'fence');
 	}
-	return;
+	if ($ns->get_node_state($sd->{node}) ne 'maintenance') {
+	    return;
+	} else {
+	    # save current node as fallback for when it comes out of
+	    # maintenance
+	    $sd->{maintenance_node} = $sd->{node};
+	}
     }
-	
+
     if ($cd->{state} eq 'disabled' || $cd->{state} eq 'stopped') {
 	&$change_service_state($self, $sid, 'request_stop');
 	return;
@@ -634,10 +665,10 @@ sub next_state_started {
     if ($cd->{state} eq 'started') {
 
 	if ($sd->{cmd}) {
-	    my ($cmd, $target) = @{$sd->{cmd}};
-	    delete $sd->{cmd};
+	    my $cmd = shift @{$sd->{cmd}};
 
 	    if ($cmd eq 'migrate' || $cmd eq 'relocate') {
+		my $target = shift @{$sd->{cmd}};
 		if (!$ns->node_is_online($target)) {
 		    $haenv->log('err', "ignore service '$sid' $cmd request - node '$target' not online");
 		} elsif ($sd->{node} eq $target) {
@@ -646,9 +677,21 @@ sub next_state_started {
 		    $haenv->log('info', "$cmd service '$sid' to node '$target'");
 		    &$change_service_state($self, $sid, $cmd, node => $sd->{node}, target => $target);
 		}
+	    } elsif ($cmd eq 'stop') {
+		my $timeout = shift @{$sd->{cmd}};
+		if ($timeout == 0) {
+		    $haenv->log('info', "request immediate service hard-stop for service '$sid'");
+		} else {
+		    $haenv->log('info', "request graceful stop with timeout '$timeout' for service '$sid'");
+		}
+		&$change_service_state($self, $sid, 'request_stop', timeout => $timeout);
+		$haenv->update_service_config($sid, {'state' => 'stopped'});
 	    } else {
-		$haenv->log('err', "unknown command '$cmd' for service '$sid'"); 
+		$haenv->log('err', "unknown command '$cmd' for service '$sid'");
 	    }
+
+	    delete $sd->{cmd};
+
 	} else {
 
 	    my $try_next = 0;
@@ -703,10 +746,29 @@ sub next_state_started {
 		}
 	    }
 
-	    my $node = select_service_node($self->{groups}, $self->{online_node_usage},
-					   $cd, $sd->{node}, $try_next, $sd->{failed_nodes});
+	    my $node = select_service_node(
+	        $self->{groups},
+		$self->{online_node_usage},
+		$cd,
+		$sd->{node},
+		$try_next,
+		$sd->{failed_nodes},
+		$sd->{maintenance_node},
+	    );
 
 	    if ($node && ($sd->{node} ne $node)) {
+		$self->{online_node_usage}->{$node}++;
+
+		if (defined(my $fallback = $sd->{maintenance_node})) {
+		    if ($node eq $fallback) {
+			$haenv->log('info', "moving service '$sid' back to '$fallback', node came back from maintenance.");
+			delete $sd->{maintenance_node};
+		    } elsif ($sd->{node} ne $fallback) {
+			$haenv->log('info', "dropping maintenance fallback node '$fallback' for '$sid'");
+			delete $sd->{maintenance_node};
+		    }
+		}
+
 		if ($cd->{type} eq 'vm') {
 		    $haenv->log('info', "migrate service '$sid' to node '$node' (running)");
 		    &$change_service_state($self, $sid, 'migrate', node => $sd->{node}, target => $node);