[pve-ha-manager.git] / src / PVE / HA / Sim / Hardware.pm

package PVE::HA::Sim::Hardware;

# Simulate Hardware resources

# power supply for nodes: on/off
# network connection to nodes: on/off
# watchdog devices for nodes

use strict;
use warnings;
use POSIX qw(strftime EINTR);
use JSON; 
use IO::File;
use Fcntl qw(:DEFAULT :flock);
use File::Copy;
use File::Path qw(make_path remove_tree);
use PVE::HA::Config;
use PVE::HA::FenceConfig;

my $watchdog_timeout = 60;


# Status directory layout
#
# configuration
#
# $testdir/cmdlist                    Command list for simulation
# $testdir/hardware_status            Hardware description (number of nodes, ...)
# $testdir/manager_status             CRM status (start with {})
# $testdir/service_config             Service configuration
# $testdir/groups                     HA groups configuration
# $testdir/service_status_<node>      Service status

#
# runtime status for simulation system
#
# $testdir/status/cluster_locks        Cluster locks
# $testdir/status/hardware_status      Hardware status (power/network on/off)
# $testdir/status/watchdog_status      Watchdog status
#
# runtime status
#
# $testdir/status/lrm_status_<node>           LRM status
# $testdir/status/manager_status              CRM status
# $testdir/status/crm_commands                CRM command queue
# $testdir/status/service_config              Service configuration
# $testdir/status/service_status_<node>       Service status
# $testdir/status/groups                      HA groups configuration

sub read_lrm_status {
    my ($self, $node) = @_;

    my $filename = "$self->{statusdir}/lrm_status_$node";

    return PVE::HA::Tools::read_json_from_file($filename, {});  
}

sub write_lrm_status {
    my ($self, $node, $status_obj) = @_;

    my $filename = "$self->{statusdir}/lrm_status_$node";

    PVE::HA::Tools::write_json_to_file($filename, $status_obj); 
}

sub read_hardware_status_nolock {
    my ($self) = @_;

    my $filename = "$self->{statusdir}/hardware_status";

    my $raw = PVE::Tools::file_get_contents($filename);
    my $cstatus = decode_json($raw);

    return $cstatus;
}

sub write_hardware_status_nolock {
    my ($self, $cstatus) = @_;

    my $filename = "$self->{statusdir}/hardware_status";

    PVE::Tools::file_set_contents($filename, encode_json($cstatus));
};

sub read_service_config {
    my ($self) = @_;

    my $filename = "$self->{statusdir}/service_config";
    my $conf = PVE::HA::Tools::read_json_from_file($filename); 

    foreach my $sid (keys %$conf) {
	my $d = $conf->{$sid};

	die "service '$sid' without assigned node!" if !$d->{node};

	if ($sid =~ m/^(vm|ct|fa):(\d+)$/) {
	    $d->{type} = $1;
	    $d->{name} = $2;
	} else {
	    die "implement me";
	}
	$d->{state} = 'disabled' if !$d->{state};
	$d->{state} = 'started' if $d->{state} eq 'enabled'; # backward compatibility
	$d->{max_restart} = 1 if !defined($d->{max_restart});
	$d->{max_relocate} = 1 if !defined($d->{max_relocate});
    }

    return $conf;
}

sub write_service_config {
    my ($self, $conf) = @_;

    $self->{service_config} = $conf;

    my $filename = "$self->{statusdir}/service_config";
    return PVE::HA::Tools::write_json_to_file($filename, $conf);
}

sub read_fence_config {
    my ($self) = @_;

    my $raw = undef;

    my $filename = "$self->{statusdir}/fence.cfg";
    if (-e $filename) {
	$raw = PVE::Tools::file_get_contents($filename);
    }

    return PVE::HA::FenceConfig::parse_config($filename, $raw);
}

sub exec_fence_agent {
    my ($self, $agent, $node, @param) = @_;

    # let all agent succeed and behave the same for now
    $self->sim_hardware_cmd("power $node off", $agent);

    return 0; # EXIT_SUCCESS
}

sub set_service_state {
    my ($self, $sid, $state) = @_;

    my $conf = $self->read_service_config();
    die "no such service '$sid'" if !$conf->{$sid};

    $conf->{$sid}->{state} = $state;

    $self->write_service_config($conf);

    return $conf;
}

sub add_service {
    my ($self, $sid, $opts) = @_;

    my $conf = $self->read_service_config();
    die "resource ID '$sid' already defined\n" if $conf->{$sid};

    $conf->{$sid} = $opts;

    $self->write_service_config($conf);

    return $conf;
}

sub delete_service {
    my ($self, $sid) = @_;

    my $conf = $self->read_service_config();

    die "no such service '$sid'" if !$conf->{$sid};

    delete $conf->{$sid};

    $self->write_service_config($conf);

    return $conf;
}

sub change_service_location {
    my ($self, $sid, $current_node, $new_node) = @_;

    my $conf = $self->read_service_config();

    die "no such service '$sid'\n" if !$conf->{$sid};

    die "current_node for '$sid' does not match ($current_node != $conf->{$sid}->{node})\n" 
	if $current_node ne $conf->{$sid}->{node};
    
    $conf->{$sid}->{node} = $new_node;

    $self->write_service_config($conf);
}

sub service_has_lock {
    my ($self, $sid) = @_;

    my $conf = $self->read_service_config();

    die "no such service '$sid'\n" if !$conf->{$sid};

    return $conf->{$sid}->{lock};
}

sub lock_service {
    my ($self, $sid, $lock) = @_;

    my $conf = $self->read_service_config();

    die "no such service '$sid'\n" if !$conf->{$sid};

    $conf->{$sid}->{lock} = $lock || 'backup';

    $self->write_service_config($conf);

    return $conf;
}

sub unlock_service {
    my ($self, $sid, $lock) = @_;

    my $conf = $self->read_service_config();

    die "no such service '$sid'\n" if !$conf->{$sid};

    if (!defined($conf->{$sid}->{lock})) {
	return undef;
    }

    if (defined($lock) && $conf->{$sid}->{lock} ne $lock) {
	warn "found lock '$conf->{$sid}->{lock}' trying to remove '$lock' lock\n";
	return undef;
    }

    my $removed_lock = delete $conf->{$sid}->{lock};

    $self->write_service_config($conf);

    return $removed_lock;
}

sub queue_crm_commands_nolock {
    my ($self, $cmd) = @_;

    chomp $cmd;

    my $data = '';
    my $filename = "$self->{statusdir}/crm_commands";
    if (-f $filename) {
	$data = PVE::Tools::file_get_contents($filename);
    }
    $data .= "$cmd\n";
    PVE::Tools::file_set_contents($filename, $data);

    return undef;
}

sub queue_crm_commands {
    my ($self, $cmd) = @_;

    my $code = sub { $self->queue_crm_commands_nolock($cmd); };
 
    $self->global_lock($code);

    return undef;
}

sub read_crm_commands {
    my ($self) = @_;

    my $code = sub {
	my $data = '';

 	my $filename = "$self->{statusdir}/crm_commands";
	if (-f $filename) {
	    $data = PVE::Tools::file_get_contents($filename);
	}
	PVE::Tools::file_set_contents($filename, '');

	return $data;
    };
 
    return $self->global_lock($code);
}

sub read_group_config {
    my ($self) = @_;

    my $filename = "$self->{statusdir}/groups";
    my $raw = '';
    $raw = PVE::Tools::file_get_contents($filename) if -f $filename;

    return PVE::HA::Config::parse_groups_config($filename, $raw);
}

sub read_service_status {
    my ($self, $node) = @_;

    my $filename = "$self->{statusdir}/service_status_$node";
    return PVE::HA::Tools::read_json_from_file($filename); 
}

sub write_service_status {
    my ($self, $node, $data) = @_;

    my $filename = "$self->{statusdir}/service_status_$node";
    my $res = PVE::HA::Tools::write_json_to_file($filename, $data);

    # fixme: add test if a service runs on two nodes!!!

    return $res;
} 

my $default_group_config = <<__EOD;
group: prefer_node1
    nodes node1
    nofailback 1

group: prefer_node2
    nodes node2
    nofailback 1

group: prefer_node3
    nodes node3
    nofailback 1
__EOD

sub new {
    my ($this, $testdir) = @_;

    die "missing testdir" if !$testdir;

    die "testdir '$testdir' does not exist or is not a directory!\n"
        if !-d $testdir;

    my $class = ref($this) || $this;

    my $self = bless {}, $class;

    my $statusdir = $self->{statusdir} = "$testdir/status";

    remove_tree($statusdir);
    mkdir $statusdir;

    # copy initial configuartion
    copy("$testdir/manager_status", "$statusdir/manager_status"); # optional

    if (-f "$testdir/groups") {
	copy("$testdir/groups", "$statusdir/groups");
    } else {
	PVE::Tools::file_set_contents("$statusdir/groups", $default_group_config);
    }

    if (-f "$testdir/service_config") {
	copy("$testdir/service_config", "$statusdir/service_config");
    } else {
	my $conf = {
	    'vm:101' => { node => 'node1', group => 'prefer_node1' },
	    'vm:102' => { node => 'node2', group => 'prefer_node2' },
	    'vm:103' => { node => 'node3', group => 'prefer_node3' },
	    'vm:104' => { node => 'node1', group => 'prefer_node1' },
	    'vm:105' => { node => 'node2', group => 'prefer_node2' },
	    'vm:106' => { node => 'node3', group => 'prefer_node3' },
	};
	$self->write_service_config($conf);
    }

    if (-f "$testdir/hardware_status") {
	copy("$testdir/hardware_status", "$statusdir/hardware_status") ||
	    die "Copy failed: $!\n";
    } else {
	my $cstatus = {
	    node1 => { power => 'off', network => 'off' },
	    node2 => { power => 'off', network => 'off' },
	    node3 => { power => 'off', network => 'off' },
	};
	$self->write_hardware_status_nolock($cstatus);
    }

    if (-f "$testdir/fence.cfg") {
	copy("$testdir/fence.cfg", "$statusdir/fence.cfg");
    }

    my $cstatus = $self->read_hardware_status_nolock();

    foreach my $node (sort keys %$cstatus) {
	$self->{nodes}->{$node} = {};

	if (-f "$testdir/service_status_$node") {
	    copy("$testdir/service_status_$node", "$statusdir/service_status_$node");
	} else {	
	    $self->write_service_status($node, {});
	}
    }

    $self->{service_config} = $self->read_service_config();

    return $self;
}

sub get_time {
    my ($self) = @_;

    die "implement in subclass";
}

sub log {
    my ($self, $level, $msg, $id) = @_;

    chomp $msg;

    my $time = $self->get_time();

    $id = 'hardware' if !$id;

    printf("%-5s %5d %12s: $msg\n", $level, $time, $id);
}

sub statusdir {
    my ($self, $node) = @_;

    return $self->{statusdir};
}

sub global_lock {
    my ($self, $code, @param) = @_;

    my $lockfile = "$self->{statusdir}/hardware.lck";
    my $fh = IO::File->new(">>$lockfile") ||
	die "unable to open '$lockfile'\n";

    my $success;
    for (;;) {
	$success = flock($fh, LOCK_EX);
	if ($success || ($! != EINTR)) {
	    last;
	}
	if (!$success) {
	    close($fh);
	    die "can't acquire lock '$lockfile' - $!\n";
	}
    }

    my $res;

    eval { $res = &$code($fh, @param) };
    my $err = $@;
    
    close($fh);

    die $err if $err;
    
    return $res;
}

my $compute_node_info = sub {
    my ($self, $cstatus) = @_;

    my $node_info = {};

    my $node_count = 0;
    my $online_count = 0;

    foreach my $node (keys %$cstatus) {
	my $d = $cstatus->{$node};

	my $online = ($d->{power} eq 'on' && $d->{network} eq 'on') ? 1 : 0;
	$node_info->{$node}->{online} = $online;

	$node_count++;
	$online_count++ if $online;
    }

    my $quorate = ($online_count > int($node_count/2)) ? 1 : 0;
		   
    if (!$quorate) {
	foreach my $node (keys %$cstatus) {
	    my $d = $cstatus->{$node};
	    $node_info->{$node}->{online} = 0;
	}
    }

    return ($node_info, $quorate);
};

sub get_node_info {
    my ($self) = @_;

    my $cstatus = $self->read_hardware_status_nolock();
    my ($node_info, $quorate) = &$compute_node_info($self, $cstatus);

    return ($node_info, $quorate);
}

# simulate hardware commands
# power <node> <on|off>
# network <node> <on|off>
# reboot <node>
# shutdown <node>
# restart-lrm <node>
# service <sid> <started|disabled|stopped>
# service <sid> <migrate|relocate> <target>
# service <sid> lock/unlock [lockname]

sub sim_hardware_cmd {
    my ($self, $cmdstr, $logid) = @_;

    my $code = sub {
	my ($lock_fh) = @_;

	my $cstatus = $self->read_hardware_status_nolock();

	my ($cmd, $objid, $action, $target) = split(/\s+/, $cmdstr);

	die "sim_hardware_cmd: no node or service for command specified"
	    if !$objid;

	my ($node, $sid, $d);

	if ($cmd eq 'service') {
	    $sid = PVE::HA::Tools::pve_verify_ha_resource_id($objid);
	} else {
	    $node = $objid;
	    $d = $self->{nodes}->{$node} ||
		die "sim_hardware_cmd: no such node '$node'\n";
	}

	$self->log('info', "execute $cmdstr", $logid);

	if ($cmd eq 'power') {
	    die "sim_hardware_cmd: unknown action '$action'\n"
		if $action !~ m/^(on|off)$/;

	    if ($cstatus->{$node}->{power} ne $action) {
		if ($action eq 'on') {

		    $d->{crm} = $self->crm_control('start', $d, $lock_fh) if !defined($d->{crm});
		    $d->{lrm} = $self->lrm_control('start', $d, $lock_fh) if !defined($d->{lrm});
		    $d->{lrm_restart} = undef;

		} else {

		    if ($d->{crm}) {
			$d->{crm_env}->log('info', "killed by poweroff");
			$self->crm_control('stop', $d, $lock_fh);
			$d->{crm} = undef;
		    }
		    if ($d->{lrm}) {
			$d->{lrm_env}->log('info', "killed by poweroff");
			$self->lrm_control('stop', $d, $lock_fh);
			$d->{lrm} = undef;
			$d->{lrm_restart} = undef;
		    }

		    $self->watchdog_reset_nolock($node);
		    $self->write_service_status($node, {});
		}
	    }

	    $cstatus->{$node}->{power} = $action;
	    $cstatus->{$node}->{network} = $action;
	    $cstatus->{$node}->{shutdown} = undef;

	    $self->write_hardware_status_nolock($cstatus);

	} elsif ($cmd eq 'network') {
	    die "sim_hardware_cmd: unknown network action '$action'"
		if $action !~ m/^(on|off)$/;
	    $cstatus->{$node}->{network} = $action;

	    $self->write_hardware_status_nolock($cstatus);

	} elsif ($cmd eq 'reboot' || $cmd eq 'shutdown') {
	    $cstatus->{$node}->{shutdown} = $cmd;

	    $self->write_hardware_status_nolock($cstatus);

	    $self->lrm_control('shutdown', $d, $lock_fh) if defined($d->{lrm});
	} elsif ($cmd eq 'restart-lrm') {
	    if ($d->{lrm}) {
		$d->{lrm_restart} = 1;
		$self->lrm_control('shutdown', $d, $lock_fh);
	    }
	} elsif ($cmd eq 'crm') {

	    if ($action eq 'stop') {
		if ($d->{crm}) {
		    $d->{crm_stop} = 1;
		    $self->crm_control('shutdown', $d, $lock_fh);
		}
	    } elsif ($action eq 'start') {
		$d->{crm} = $self->crm_control('start', $d, $lock_fh) if !defined($d->{crm});
	    } else {
		die "sim_hardware_cmd: unknown action '$action'";
	    }

	} elsif ($cmd eq 'service') {
	    if ($action eq 'started' || $action eq 'disabled' || $action eq 'stopped') {

		$self->set_service_state($sid, $action);

	    } elsif ($action eq 'migrate' || $action eq 'relocate') {

		die "sim_hardware_cmd: missing target node for '$action' command"
		    if !$target;

		$self->queue_crm_commands_nolock("$action $sid $target");

	    } elsif ($action eq 'add') {

		$self->add_service($sid, {state => 'started', node => $target});

	    } elsif ($action eq 'delete') {

		$self->delete_service($sid);

	    } elsif ($action eq 'lock') {

		$self->lock_service($sid, $target);

	    } elsif ($action eq 'unlock') {

		$self->unlock_service($sid, $target);

	    } else {
		die "sim_hardware_cmd: unknown service action '$action' " .
		    "- not implemented\n"
	    }
	} else {
	    die "sim_hardware_cmd: unknown command '$cmdstr'\n";
	}

	return $cstatus;
    };

    return $self->global_lock($code);
}

# for controlling the resource manager services
sub crm_control {
    my ($self, $action, $data, $lock_fh) = @_;

    die "implement in subclass";
}

sub lrm_control {
    my ($self, $action, $data, $lock_fh) = @_;

    die "implement in subclass";
}

sub run {
    my ($self) = @_;

    die "implement in subclass";
}

my $modify_watchog = sub {
    my ($self, $code) = @_;

    my $update_cmd = sub {

	my $filename = "$self->{statusdir}/watchdog_status";
 
	my ($res, $wdstatus);

	if (-f $filename) {
	    my $raw = PVE::Tools::file_get_contents($filename);
	    $wdstatus = decode_json($raw);
	} else {
	    $wdstatus = {};
	}
	
	($wdstatus, $res) = &$code($wdstatus);

	PVE::Tools::file_set_contents($filename, encode_json($wdstatus));

	return $res;
    };

    return $self->global_lock($update_cmd);
};

sub watchdog_reset_nolock {
    my ($self, $node) = @_;

    my $filename = "$self->{statusdir}/watchdog_status";

    if (-f $filename) {
 	my $raw = PVE::Tools::file_get_contents($filename);
	my $wdstatus = decode_json($raw);

	foreach my $id (keys %$wdstatus) {
	    delete $wdstatus->{$id} if $wdstatus->{$id}->{node} eq $node;
	}
	
	PVE::Tools::file_set_contents($filename, encode_json($wdstatus));
    }
}

sub watchdog_check {
    my ($self, $node) = @_;

    my $code = sub {
	my ($wdstatus) = @_;

	my $res = 1;

	foreach my $wfh (keys %$wdstatus) {
	    my $wd = $wdstatus->{$wfh};
	    next if $wd->{node} ne $node;

	    my $ctime = $self->get_time();
	    my $tdiff = $ctime - $wd->{update_time};

	    if ($tdiff > $watchdog_timeout) { # expired
		$res = 0;
		delete $wdstatus->{$wfh};
	    }
	}
	
	return ($wdstatus, $res);
    };

    return &$modify_watchog($self, $code);
}

my $wdcounter = 0;

sub watchdog_open {
    my ($self, $node) = @_;

    my $code = sub {
	my ($wdstatus) = @_;

	++$wdcounter;

	my $id = "WD:$node:$$:$wdcounter";

	die "internal error" if defined($wdstatus->{$id});

	$wdstatus->{$id} = {
	    node => $node,
	    update_time => $self->get_time(),
	};

	return ($wdstatus, $id);
    };

    return &$modify_watchog($self, $code);
}

sub watchdog_close {
    my ($self, $wfh) = @_;

    my $code = sub {
	my ($wdstatus) = @_;

	my $wd = $wdstatus->{$wfh};
	die "no such watchdog handle '$wfh'\n" if !defined($wd);

	my $tdiff = $self->get_time() - $wd->{update_time};
	die "watchdog expired" if $tdiff > $watchdog_timeout;

	delete $wdstatus->{$wfh};

	return ($wdstatus);
    };

    return &$modify_watchog($self, $code);
}

sub watchdog_update {
    my ($self, $wfh) = @_;

    my $code = sub {
	my ($wdstatus) = @_;

	my $wd = $wdstatus->{$wfh};

	die "no such watchdog handle '$wfh'\n" if !defined($wd);

	my $ctime = $self->get_time();
	my $tdiff = $ctime - $wd->{update_time};

	die "watchdog expired" if $tdiff > $watchdog_timeout;
	
	$wd->{update_time} = $ctime;

	return ($wdstatus);
    };

    return &$modify_watchog($self, $code);
}

1;
Commit	Line	Data
8b3f9144 DM	1	package PVE::HA::Sim::Hardware;
	2
	3	# Simulate Hardware resources
	4
	5	# power supply for nodes: on/off
	6	# network connection to nodes: on/off
	7	# watchdog devices for nodes
0cfd8f5b DM	8
	9	use strict;
	10	use warnings;
	11	use POSIX qw(strftime EINTR);
0cfd8f5b DM	12	use JSON;
	13	use IO::File;
	14	use Fcntl qw(:DEFAULT :flock);
787b66eb DM	15	use File::Copy;
787b66eb DM	16	use File::Path qw(make_path remove_tree);
c982dfee TL	17	use PVE::HA::Config;
c982dfee TL	18	use PVE::HA::FenceConfig;
f5a14b93	19
17b5cf98	20	my $watchdog_timeout = 60;
0bba8f60	21
0cfd8f5b	22
787b66eb DM	23	# Status directory layout
	24	#
	25	# configuration
	26	#
8456bde2 DM	27	# $testdir/cmdlist Command list for simulation
	28	# $testdir/hardware_status Hardware description (number of nodes, ...)
	29	# $testdir/manager_status CRM status (start with {})
	30	# $testdir/service_config Service configuration
abc920b4	31	# $testdir/groups HA groups configuration
8456bde2	32	# $testdir/service_status_<node> Service status
3c36cbca	33
9329c1e2 DM	34	#
	35	# runtime status for simulation system
	36	#
	37	# $testdir/status/cluster_locks Cluster locks
	38	# $testdir/status/hardware_status Hardware status (power/network on/off)
	39	# $testdir/status/watchdog_status Watchdog status
787b66eb DM	40	#
787b66eb DM	41	# runtime status
9329c1e2	42	#
8456bde2 DM	43	# $testdir/status/lrm_status_<node> LRM status
8456bde2 DM	44	# $testdir/status/manager_status CRM status
abc920b4	45	# $testdir/status/crm_commands CRM command queue
8456bde2 DM	46	# $testdir/status/service_config Service configuration
8456bde2 DM	47	# $testdir/status/service_status_<node> Service status
abc920b4	48	# $testdir/status/groups HA groups configuration
c4a221bc DM	49
	50	sub read_lrm_status {
	51	my ($self, $node) = @_;
	52
	53	my $filename = "$self->{statusdir}/lrm_status_$node";
	54
	55	return PVE::HA::Tools::read_json_from_file($filename, {});
	56	}
	57
	58	sub write_lrm_status {
	59	my ($self, $node, $status_obj) = @_;
	60
	61	my $filename = "$self->{statusdir}/lrm_status_$node";
	62
	63	PVE::HA::Tools::write_json_to_file($filename, $status_obj);
	64	}
787b66eb	65
8b3f9144	66	sub read_hardware_status_nolock {
0cfd8f5b DM	67	my ($self) = @_;
0cfd8f5b DM	68
8b3f9144	69	my $filename = "$self->{statusdir}/hardware_status";
0cfd8f5b DM	70
	71	my $raw = PVE::Tools::file_get_contents($filename);
	72	my $cstatus = decode_json($raw);
	73
	74	return $cstatus;
	75	}
	76
8b3f9144	77	sub write_hardware_status_nolock {
0cfd8f5b DM	78	my ($self, $cstatus) = @_;
0cfd8f5b DM	79
8b3f9144	80	my $filename = "$self->{statusdir}/hardware_status";
0cfd8f5b DM	81
	82	PVE::Tools::file_set_contents($filename, encode_json($cstatus));
	83	};
	84
95360669 DM	85	sub read_service_config {
	86	my ($self) = @_;
	87
	88	my $filename = "$self->{statusdir}/service_config";
	89	my $conf = PVE::HA::Tools::read_json_from_file($filename);
	90
	91	foreach my $sid (keys %$conf) {
	92	my $d = $conf->{$sid};
8456bde2 DM	93
	94	die "service '$sid' without assigned node!" if !$d->{node};
	95
ec368d74	96	if ($sid =~ m/^(vm\|ct\|fa):(\d+)$/) {
b026c8c9 DM	97	$d->{type} = $1;
b026c8c9 DM	98	$d->{name} = $2;
95360669 DM	99	} else {
	100	die "implement me";
	101	}
	102	$d->{state} = 'disabled' if !$d->{state};
bb07bd2c	103	$d->{state} = 'started' if $d->{state} eq 'enabled'; # backward compatibility
ec368d74 TL	104	$d->{max_restart} = 1 if !defined($d->{max_restart});
ec368d74 TL	105	$d->{max_relocate} = 1 if !defined($d->{max_relocate});
95360669 DM	106	}
	107
	108	return $conf;
	109	}
	110
79e0e005 DM	111	sub write_service_config {
	112	my ($self, $conf) = @_;
	113
95360669 DM	114	$self->{service_config} = $conf;
95360669 DM	115
79e0e005 DM	116	my $filename = "$self->{statusdir}/service_config";
79e0e005 DM	117	return PVE::HA::Tools::write_json_to_file($filename, $conf);
c982dfee TL	118	}
	119
	120	sub read_fence_config {
	121	my ($self) = @_;
	122
	123	my $raw = undef;
	124
	125	my $filename = "$self->{statusdir}/fence.cfg";
	126	if (-e $filename) {
	127	$raw = PVE::Tools::file_get_contents($filename);
	128	}
	129
	130	return PVE::HA::FenceConfig::parse_config($filename, $raw);
	131	}
	132
	133	sub exec_fence_agent {
	134	my ($self, $agent, $node, @param) = @_;
	135
	136	# let all agent succeed and behave the same for now
	137	$self->sim_hardware_cmd("power $node off", $agent);
	138
	139	return 0; # EXIT_SUCCESS
	140	}
79e0e005	141
e5f43426 TL	142	sub set_service_state {
	143	my ($self, $sid, $state) = @_;
	144
	145	my $conf = $self->read_service_config();
	146	die "no such service '$sid'" if !$conf->{$sid};
	147
	148	$conf->{$sid}->{state} = $state;
	149
	150	$self->write_service_config($conf);
	151
	152	return $conf;
	153	}
	154
27ccc95c TL	155	sub add_service {
	156	my ($self, $sid, $opts) = @_;
	157
	158	my $conf = $self->read_service_config();
	159	die "resource ID '$sid' already defined\n" if $conf->{$sid};
	160
	161	$conf->{$sid} = $opts;
	162
	163	$self->write_service_config($conf);
	164
	165	return $conf;
	166	}
	167
	168	sub delete_service {
	169	my ($self, $sid) = @_;
	170
	171	my $conf = $self->read_service_config();
	172
	173	die "no such service '$sid'" if !$conf->{$sid};
	174
	175	delete $conf->{$sid};
	176
	177	$self->write_service_config($conf);
	178
	179	return $conf;
	180	}
	181
8456bde2	182	sub change_service_location {
6da27e23	183	my ($self, $sid, $current_node, $new_node) = @_;
8456bde2 DM	184
	185	my $conf = $self->read_service_config();
	186
	187	die "no such service '$sid'\n" if !$conf->{$sid};
	188
6da27e23 DM	189	die "current_node for '$sid' does not match ($current_node != $conf->{$sid}->{node})\n"
	190	if $current_node ne $conf->{$sid}->{node};
	191
	192	$conf->{$sid}->{node} = $new_node;
8456bde2 DM	193
	194	$self->write_service_config($conf);
	195	}
	196
cde11324 TL	197	sub service_has_lock {
	198	my ($self, $sid) = @_;
	199
	200	my $conf = $self->read_service_config();
	201
	202	die "no such service '$sid'\n" if !$conf->{$sid};
	203
	204	return $conf->{$sid}->{lock};
	205	}
	206
	207	sub lock_service {
	208	my ($self, $sid, $lock) = @_;
	209
	210	my $conf = $self->read_service_config();
	211
	212	die "no such service '$sid'\n" if !$conf->{$sid};
	213
	214	$conf->{$sid}->{lock} = $lock \|\| 'backup';
	215
	216	$self->write_service_config($conf);
	217
	218	return $conf;
	219	}
	220
	221	sub unlock_service {
	222	my ($self, $sid, $lock) = @_;
	223
	224	my $conf = $self->read_service_config();
	225
	226	die "no such service '$sid'\n" if !$conf->{$sid};
	227
	228	if (!defined($conf->{$sid}->{lock})) {
cde11324 TL	229	return undef;
	230	}
	231
	232	if (defined($lock) && $conf->{$sid}->{lock} ne $lock) {
	233	warn "found lock '$conf->{$sid}->{lock}' trying to remove '$lock' lock\n";
	234	return undef;
	235	}
	236
	237	my $removed_lock = delete $conf->{$sid}->{lock};
	238
	239	$self->write_service_config($conf);
	240
	241	return $removed_lock;
	242	}
	243
b70aa69e	244	sub queue_crm_commands_nolock {
3b996922 DM	245	my ($self, $cmd) = @_;
	246
	247	chomp $cmd;
	248
b70aa69e DM	249	my $data = '';
	250	my $filename = "$self->{statusdir}/crm_commands";
	251	if (-f $filename) {
	252	$data = PVE::Tools::file_get_contents($filename);
	253	}
	254	$data .= "$cmd\n";
	255	PVE::Tools::file_set_contents($filename, $data);
	256
	257	return undef;
	258	}
	259
	260	sub queue_crm_commands {
	261	my ($self, $cmd) = @_;
	262
	263	my $code = sub { $self->queue_crm_commands_nolock($cmd); };
3b996922 DM	264
	265	$self->global_lock($code);
	266
	267	return undef;
	268	}
	269
	270	sub read_crm_commands {
	271	my ($self) = @_;
	272
	273	my $code = sub {
	274	my $data = '';
	275
	276	my $filename = "$self->{statusdir}/crm_commands";
	277	if (-f $filename) {
	278	$data = PVE::Tools::file_get_contents($filename);
	279	}
	280	PVE::Tools::file_set_contents($filename, '');
	281
	282	return $data;
	283	};
	284
	285	return $self->global_lock($code);
	286	}
	287
abc920b4 DM	288	sub read_group_config {
	289	my ($self) = @_;
	290
	291	my $filename = "$self->{statusdir}/groups";
	292	my $raw = '';
	293	$raw = PVE::Tools::file_get_contents($filename) if -f $filename;
	294
cc32b737	295	return PVE::HA::Config::parse_groups_config($filename, $raw);
abc920b4 DM	296	}
abc920b4 DM	297
c4a221bc	298	sub read_service_status {
8456bde2	299	my ($self, $node) = @_;
c4a221bc	300
8456bde2	301	my $filename = "$self->{statusdir}/service_status_$node";
c4a221bc DM	302	return PVE::HA::Tools::read_json_from_file($filename);
	303	}
	304
	305	sub write_service_status {
8456bde2 DM	306	my ($self, $node, $data) = @_;
	307
	308	my $filename = "$self->{statusdir}/service_status_$node";
	309	my $res = PVE::HA::Tools::write_json_to_file($filename, $data);
	310
	311	# fixme: add test if a service runs on two nodes!!!
c4a221bc	312
8456bde2	313	return $res;
c4a221bc DM	314	}
c4a221bc DM	315
abc920b4 DM	316	my $default_group_config = <<__EOD;
	317	group: prefer_node1
	318	nodes node1
e941bdc5	319	nofailback 1
abc920b4 DM	320
	321	group: prefer_node2
	322	nodes node2
e941bdc5	323	nofailback 1
abc920b4 DM	324
abc920b4 DM	325	group: prefer_node3
7a294ad4	326	nodes node3
e941bdc5	327	nofailback 1
abc920b4 DM	328	__EOD
abc920b4 DM	329
0cfd8f5b DM	330	sub new {
	331	my ($this, $testdir) = @_;
	332
	333	die "missing testdir" if !$testdir;
	334
ba9e808e TL	335	die "testdir '$testdir' does not exist or is not a directory!\n"
	336	if !-d $testdir;
	337
0cfd8f5b DM	338	my $class = ref($this) \|\| $this;
	339
	340	my $self = bless {}, $class;
	341
787b66eb DM	342	my $statusdir = $self->{statusdir} = "$testdir/status";
	343
	344	remove_tree($statusdir);
	345	mkdir $statusdir;
0cfd8f5b	346
787b66eb DM	347	# copy initial configuartion
787b66eb DM	348	copy("$testdir/manager_status", "$statusdir/manager_status"); # optional
79e0e005	349
abc920b4 DM	350	if (-f "$testdir/groups") {
	351	copy("$testdir/groups", "$statusdir/groups");
	352	} else {
	353	PVE::Tools::file_set_contents("$statusdir/groups", $default_group_config);
	354	}
	355
79e0e005 DM	356	if (-f "$testdir/service_config") {
	357	copy("$testdir/service_config", "$statusdir/service_config");
	358	} else {
	359	my $conf = {
eda9314d DM	360	'vm:101' => { node => 'node1', group => 'prefer_node1' },
	361	'vm:102' => { node => 'node2', group => 'prefer_node2' },
	362	'vm:103' => { node => 'node3', group => 'prefer_node3' },
	363	'vm:104' => { node => 'node1', group => 'prefer_node1' },
	364	'vm:105' => { node => 'node2', group => 'prefer_node2' },
	365	'vm:106' => { node => 'node3', group => 'prefer_node3' },
79e0e005 DM	366	};
	367	$self->write_service_config($conf);
	368	}
787b66eb	369
853f5867 DM	370	if (-f "$testdir/hardware_status") {
	371	copy("$testdir/hardware_status", "$statusdir/hardware_status") \|\|
	372	die "Copy failed: $!\n";
	373	} else {
	374	my $cstatus = {
	375	node1 => { power => 'off', network => 'off' },
	376	node2 => { power => 'off', network => 'off' },
	377	node3 => { power => 'off', network => 'off' },
	378	};
	379	$self->write_hardware_status_nolock($cstatus);
	380	}
787b66eb	381
c982dfee TL	382	if (-f "$testdir/fence.cfg") {
	383	copy("$testdir/fence.cfg", "$statusdir/fence.cfg");
	384	}
0cfd8f5b	385
8b3f9144	386	my $cstatus = $self->read_hardware_status_nolock();
0cfd8f5b DM	387
0cfd8f5b DM	388	foreach my $node (sort keys %$cstatus) {
0bba8f60	389	$self->{nodes}->{$node} = {};
8456bde2 DM	390
	391	if (-f "$testdir/service_status_$node") {
	392	copy("$testdir/service_status_$node", "$statusdir/service_status_$node");
	393	} else {
	394	$self->write_service_status($node, {});
	395	}
0cfd8f5b DM	396	}
0cfd8f5b DM	397
95360669 DM	398	$self->{service_config} = $self->read_service_config();
95360669 DM	399
0cfd8f5b DM	400	return $self;
	401	}
	402
	403	sub get_time {
	404	my ($self) = @_;
	405
bf93e2a2	406	die "implement in subclass";
0cfd8f5b DM	407	}
	408
	409	sub log {
fde8362a	410	my ($self, $level, $msg, $id) = @_;
0cfd8f5b DM	411
	412	chomp $msg;
	413
	414	my $time = $self->get_time();
	415
fde8362a DM	416	$id = 'hardware' if !$id;
fde8362a DM	417
0bba8f60	418	printf("%-5s %5d %12s: $msg\n", $level, $time, $id);
0cfd8f5b DM	419	}
	420
	421	sub statusdir {
	422	my ($self, $node) = @_;
	423
	424	return $self->{statusdir};
	425	}
	426
8b3f9144	427	sub global_lock {
0cfd8f5b DM	428	my ($self, $code, @param) = @_;
0cfd8f5b DM	429
8b3f9144	430	my $lockfile = "$self->{statusdir}/hardware.lck";
0cfd8f5b DM	431	my $fh = IO::File->new(">>$lockfile") \|\|
	432	die "unable to open '$lockfile'\n";
	433
	434	my $success;
	435	for (;;) {
	436	$success = flock($fh, LOCK_EX);
	437	if ($success \|\| ($! != EINTR)) {
	438	last;
	439	}
	440	if (!$success) {
9de9a6ce	441	close($fh);
63f6a08c	442	die "can't acquire lock '$lockfile' - $!\n";
0cfd8f5b DM	443	}
0cfd8f5b DM	444	}
9de9a6ce	445
0cfd8f5b DM	446	my $res;
0cfd8f5b DM	447
9de9a6ce	448	eval { $res = &$code($fh, @param) };
0cfd8f5b	449	my $err = $@;
9de9a6ce	450
0cfd8f5b DM	451	close($fh);
	452
	453	die $err if $err;
	454
	455	return $res;
	456	}
	457
8b3f9144 DM	458	my $compute_node_info = sub {
	459	my ($self, $cstatus) = @_;
	460
	461	my $node_info = {};
	462
	463	my $node_count = 0;
	464	my $online_count = 0;
	465
	466	foreach my $node (keys %$cstatus) {
	467	my $d = $cstatus->{$node};
	468
	469	my $online = ($d->{power} eq 'on' && $d->{network} eq 'on') ? 1 : 0;
	470	$node_info->{$node}->{online} = $online;
	471
	472	$node_count++;
	473	$online_count++ if $online;
	474	}
	475
	476	my $quorate = ($online_count > int($node_count/2)) ? 1 : 0;
	477
	478	if (!$quorate) {
	479	foreach my $node (keys %$cstatus) {
	480	my $d = $cstatus->{$node};
	481	$node_info->{$node}->{online} = 0;
	482	}
	483	}
	484
	485	return ($node_info, $quorate);
	486	};
	487
	488	sub get_node_info {
	489	my ($self) = @_;
	490
5516f102 TL	491	my $cstatus = $self->read_hardware_status_nolock();
5516f102 TL	492	my ($node_info, $quorate) = &$compute_node_info($self, $cstatus);
8b3f9144 DM	493
	494	return ($node_info, $quorate);
	495	}
	496
	497	# simulate hardware commands
0cfd8f5b DM	498	# power <node> <on\|off>
0cfd8f5b DM	499	# network <node> <on\|off>
e08a0717 TL	500	# reboot <node>
	501	# shutdown <node>
	502	# restart-lrm <node>
	503	# service <sid> <started\|disabled\|stopped>
	504	# service <sid> <migrate\|relocate> <target>
	505	# service <sid> lock/unlock [lockname]
0cfd8f5b	506
8b3f9144	507	sub sim_hardware_cmd {
fde8362a	508	my ($self, $cmdstr, $logid) = @_;
0cfd8f5b	509
e08a0717 TL	510	my $code = sub {
	511	my ($lock_fh) = @_;
	512
	513	my $cstatus = $self->read_hardware_status_nolock();
	514
	515	my ($cmd, $objid, $action, $target) = split(/\s+/, $cmdstr);
	516
	517	die "sim_hardware_cmd: no node or service for command specified"
	518	if !$objid;
	519
	520	my ($node, $sid, $d);
	521
	522	if ($cmd eq 'service') {
	523	$sid = PVE::HA::Tools::pve_verify_ha_resource_id($objid);
	524	} else {
	525	$node = $objid;
	526	$d = $self->{nodes}->{$node} \|\|
	527	die "sim_hardware_cmd: no such node '$node'\n";
	528	}
	529
	530	$self->log('info', "execute $cmdstr", $logid);
	531
	532	if ($cmd eq 'power') {
	533	die "sim_hardware_cmd: unknown action '$action'\n"
	534	if $action !~ m/^(on\|off)$/;
	535
	536	if ($cstatus->{$node}->{power} ne $action) {
	537	if ($action eq 'on') {
	538
	539	$d->{crm} = $self->crm_control('start', $d, $lock_fh) if !defined($d->{crm});
	540	$d->{lrm} = $self->lrm_control('start', $d, $lock_fh) if !defined($d->{lrm});
	541	$d->{lrm_restart} = undef;
	542
	543	} else {
	544
	545	if ($d->{crm}) {
	546	$d->{crm_env}->log('info', "killed by poweroff");
	547	$self->crm_control('stop', $d, $lock_fh);
	548	$d->{crm} = undef;
	549	}
	550	if ($d->{lrm}) {
	551	$d->{lrm_env}->log('info', "killed by poweroff");
	552	$self->lrm_control('stop', $d, $lock_fh);
	553	$d->{lrm} = undef;
	554	$d->{lrm_restart} = undef;
	555	}
	556
	557	$self->watchdog_reset_nolock($node);
	558	$self->write_service_status($node, {});
	559	}
	560	}
	561
	562	$cstatus->{$node}->{power} = $action;
	563	$cstatus->{$node}->{network} = $action;
	564	$cstatus->{$node}->{shutdown} = undef;
	565
	566	$self->write_hardware_status_nolock($cstatus);
	567
	568	} elsif ($cmd eq 'network') {
	569	die "sim_hardware_cmd: unknown network action '$action'"
	570	if $action !~ m/^(on\|off)$/;
	571	$cstatus->{$node}->{network} = $action;
	572
	573	$self->write_hardware_status_nolock($cstatus);
574
575	} elsif ($cmd eq 'reboot' \|\| $cmd eq 'shutdown') {
576	$cstatus->{$node}->{shutdown} = $cmd;
577
578	$self->write_hardware_status_nolock($cstatus);
579
580	$self->lrm_control('shutdown', $d, $lock_fh) if defined($d->{lrm});
581	} elsif ($cmd eq 'restart-lrm') {
582	if ($d->{lrm}) {
583	$d->{lrm_restart} = 1;
584	$self->lrm_control('shutdown', $d, $lock_fh);
585	}
586	} elsif ($cmd eq 'crm') {
587
588	if ($action eq 'stop') {
589	if ($d->{crm}) {
590	$d->{crm_stop} = 1;
591	$self->crm_control('shutdown', $d, $lock_fh);
592	}
593	} elsif ($action eq 'start') {
594	$d->{crm} = $self->crm_control('start', $d, $lock_fh) if !defined($d->{crm});
595	} else {
596	die "sim_hardware_cmd: unknown action '$action'";
597	}
598
599	} elsif ($cmd eq 'service') {
600	if ($action eq 'started' \|\| $action eq 'disabled' \|\| $action eq 'stopped') {
601
602	$self->set_service_state($sid, $action);
603
604	} elsif ($action eq 'migrate' \|\| $action eq 'relocate') {
605
606	die "sim_hardware_cmd: missing target node for '$action' command"
607	if !$target;
608
609	$self->queue_crm_commands_nolock("$action $sid $target");
610
611	} elsif ($action eq 'add') {
612
613	$self->add_service($sid, {state => 'started', node => $target});
614
615	} elsif ($action eq 'delete') {
616
617	$self->delete_service($sid);
618
619	} elsif ($action eq 'lock') {
620
621	$self->lock_service($sid, $target);
622
623	} elsif ($action eq 'unlock') {
624
625	$self->unlock_service($sid, $target);
626
627	} else {
628	die "sim_hardware_cmd: unknown service action '$action' " .
629	"- not implemented\n"
630	}
631	} else {
632	die "sim_hardware_cmd: unknown command '$cmdstr'\n";
633	}
634
635	return $cstatus;
636	};
637
638	return $self->global_lock($code);
639	}
640
641	# for controlling the resource manager services
642	sub crm_control {
643	my ($self, $action, $data, $lock_fh) = @_;
644
645	die "implement in subclass";
646	}
647
648	sub lrm_control {
649	my ($self, $action, $data, $lock_fh) = @_;
650
bf93e2a2	651	die "implement in subclass";
0cfd8f5b DM	652	}
	653
	654	sub run {
	655	my ($self) = @_;
	656
bf93e2a2	657	die "implement in subclass";
0cfd8f5b	658	}
9329c1e2 DM	659
	660	my $modify_watchog = sub {
	661	my ($self, $code) = @_;
	662
	663	my $update_cmd = sub {
	664
	665	my $filename = "$self->{statusdir}/watchdog_status";
0cfd8f5b	666
9329c1e2 DM	667	my ($res, $wdstatus);
	668
	669	if (-f $filename) {
	670	my $raw = PVE::Tools::file_get_contents($filename);
	671	$wdstatus = decode_json($raw);
	672	} else {
	673	$wdstatus = {};
	674	}
	675
	676	($wdstatus, $res) = &$code($wdstatus);
	677
	678	PVE::Tools::file_set_contents($filename, encode_json($wdstatus));
	679
	680	return $res;
	681	};
	682
	683	return $self->global_lock($update_cmd);
	684	};
	685
0590c6a7 DM	686	sub watchdog_reset_nolock {
	687	my ($self, $node) = @_;
	688
	689	my $filename = "$self->{statusdir}/watchdog_status";
	690
	691	if (-f $filename) {
	692	my $raw = PVE::Tools::file_get_contents($filename);
	693	my $wdstatus = decode_json($raw);
	694
	695	foreach my $id (keys %$wdstatus) {
	696	delete $wdstatus->{$id} if $wdstatus->{$id}->{node} eq $node;
	697	}
	698
	699	PVE::Tools::file_set_contents($filename, encode_json($wdstatus));
	700	}
	701	}
	702
9329c1e2 DM	703	sub watchdog_check {
	704	my ($self, $node) = @_;
	705
	706	my $code = sub {
	707	my ($wdstatus) = @_;
	708
	709	my $res = 1;
	710
	711	foreach my $wfh (keys %$wdstatus) {
	712	my $wd = $wdstatus->{$wfh};
	713	next if $wd->{node} ne $node;
	714
	715	my $ctime = $self->get_time();
	716	my $tdiff = $ctime - $wd->{update_time};
	717
0bba8f60	718	if ($tdiff > $watchdog_timeout) { # expired
9329c1e2 DM	719	$res = 0;
	720	delete $wdstatus->{$wfh};
	721	}
	722	}
	723
	724	return ($wdstatus, $res);
	725	};
	726
	727	return &$modify_watchog($self, $code);
	728	}
	729
	730	my $wdcounter = 0;
	731
	732	sub watchdog_open {
	733	my ($self, $node) = @_;
	734
	735	my $code = sub {
	736	my ($wdstatus) = @_;
	737
	738	++$wdcounter;
	739
	740	my $id = "WD:$node:$$:$wdcounter";
	741
	742	die "internal error" if defined($wdstatus->{$id});
	743
	744	$wdstatus->{$id} = {
	745	node => $node,
	746	update_time => $self->get_time(),
	747	};
	748
	749	return ($wdstatus, $id);
	750	};
	751
	752	return &$modify_watchog($self, $code);
	753	}
	754
	755	sub watchdog_close {
	756	my ($self, $wfh) = @_;
	757
	758	my $code = sub {
	759	my ($wdstatus) = @_;
	760
	761	my $wd = $wdstatus->{$wfh};
	762	die "no such watchdog handle '$wfh'\n" if !defined($wd);
	763
	764	my $tdiff = $self->get_time() - $wd->{update_time};
0bba8f60	765	die "watchdog expired" if $tdiff > $watchdog_timeout;
9329c1e2 DM	766
	767	delete $wdstatus->{$wfh};
	768
	769	return ($wdstatus);
	770	};
	771
	772	return &$modify_watchog($self, $code);
	773	}
	774
	775	sub watchdog_update {
	776	my ($self, $wfh) = @_;
	777
	778	my $code = sub {
	779	my ($wdstatus) = @_;
	780
	781	my $wd = $wdstatus->{$wfh};
	782
	783	die "no such watchdog handle '$wfh'\n" if !defined($wd);
	784
	785	my $ctime = $self->get_time();
	786	my $tdiff = $ctime - $wd->{update_time};
	787
0bba8f60	788	die "watchdog expired" if $tdiff > $watchdog_timeout;
9329c1e2 DM	789
	790	$wd->{update_time} = $ctime;
	791
	792	return ($wdstatus);
	793	};
	794
	795	return &$modify_watchog($self, $code);
	796	}
	797
0cfd8f5b	798	1;