SIMPACKAGE=pve-ha-simulator
PKGREL=1
-DESTDIR=
-PREFIX=/usr
-BINDIR=${PREFIX}/bin
-SBINDIR=${PREFIX}/sbin
-MANDIR=${PREFIX}/share/man
-DOCDIR=${PREFIX}/share/doc/${PACKAGE}
-SIMDOCDIR=${PREFIX}/share/doc/${SIMPACKAGE}
-PODDIR=${DOCDIR}/pod
-MAN1DIR=${MANDIR}/man1/
-export PERLDIR=${PREFIX}/share/perl5
+GITVERSION:=$(shell cat .git/refs/heads/master)
ARCH:=$(shell dpkg-architecture -qDEB_BUILD_ARCH)
-GITVERSION:=$(shell cat .git/refs/heads/master)
DEB=${PACKAGE}_${VERSION}-${PKGREL}_${ARCH}.deb
SIMDEB=${SIMPACKAGE}_${VERSION}-${PKGREL}_all.deb
dinstall: deb simdeb
dpkg -i ${DEB} ${SIMDEB}
-%.1.gz: %.1.pod
- rm -f $@
- cat $<|pod2man -n $* -s 1 -r ${VERSION} -c "Proxmox Documentation"|gzip -c9 >$@
-
-pve-ha-crm.1.pod: pve-ha-crm
- perl -I. ./pve-ha-crm printmanpod >$@
-
-pve-ha-lrm.1.pod: pve-ha-lrm
- perl -I. ./pve-ha-lrm printmanpod >$@
-
-watchdog-mux: watchdog-mux.c
- gcc watchdog-mux.c -o watchdog-mux -Wall $$(pkg-config --libs --cflags libsystemd-daemon)
-
-.PHONY: install
-install: pve-ha-crm pve-ha-lrm pve-ha-crm.1.pod pve-ha-crm.1.gz pve-ha-lrm.1.pod pve-ha-lrm.1.gz
- install -d ${DESTDIR}${SBINDIR}
- install -m 0755 pve-ha-crm ${DESTDIR}${SBINDIR}
- install -m 0755 pve-ha-lrm ${DESTDIR}${SBINDIR}
- make -C PVE install
- install -d ${DESTDIR}/usr/share/man/man1
- install -d ${DESTDIR}${PODDIR}
- install -m 0644 pve-ha-crm.1.gz ${DESTDIR}/usr/share/man/man1/
- install -m 0644 pve-ha-crm.1.pod ${DESTDIR}/${PODDIR}
- install -m 0644 pve-ha-lrm.1.gz ${DESTDIR}/usr/share/man/man1/
- install -m 0644 pve-ha-lrm.1.pod ${DESTDIR}/${PODDIR}
-
-.PHONY: installsim
-installsim: pve-ha-simulator
- install -d ${DESTDIR}${SBINDIR}
- install -m 0755 pve-ha-simulator ${DESTDIR}${SBINDIR}
- make -C PVE installsim
.PHONY: simdeb ${SIMDEB}
simdeb ${SIMDEB}:
rm -rf build
mkdir build
- make DESTDIR=${CURDIR}/build PERLDIR=${PREFIX}/share/${SIMPACKAGE} installsim
- perl -I. ./pve-ha-crm verifyapi
- perl -I. ./pve-ha-lrm verifyapi
- install -d -m 0755 build/DEBIAN
- sed -e s/@@VERSION@@/${VERSION}/ -e s/@@PKGRELEASE@@/${PKGREL}/ -e s/@@ARCH@@/${ARCH}/ <simcontrol.in >build/DEBIAN/control
- install -D -m 0644 copyright build/${SIMDOCDIR}/copyright
- install -m 0644 changelog.Debian build/${SIMDOCDIR}/
- gzip -9 build/${SIMDOCDIR}/changelog.Debian
- echo "git clone git://git.proxmox.com/git/pve-storage.git\\ngit checkout ${GITVERSION}" > build/${SIMDOCDIR}/SOURCE
- dpkg-deb --build build
- mv build.deb ${SIMDEB}
- rm -rf debian
+ rsync -a src/ build
+ rsync -a simdebian/ build/debian
+ cp changelog.Debian build/debian/changelog
+ echo "git clone git://git.proxmox.com/git/pve-ha-manager.git\\ngit checkout ${GITVERSION}" > build/debian/SOURCE
+ cd build; dpkg-buildpackage -rfakeroot -b -us -uc
lintian ${SIMDEB}
.PHONY: deb ${DEB}
deb ${DEB}:
rm -rf build
mkdir build
- make DESTDIR=${CURDIR}/build install
- perl -I. ./pve-ha-crm verifyapi
- perl -I. ./pve-ha-lrm verifyapi
- install -d -m 0755 build/DEBIAN
- sed -e s/@@VERSION@@/${VERSION}/ -e s/@@PKGRELEASE@@/${PKGREL}/ -e s/@@ARCH@@/${ARCH}/ <control.in >build/DEBIAN/control
- install -D -m 0644 copyright build/${DOCDIR}/copyright
- install -m 0644 changelog.Debian build/${DOCDIR}/
- gzip -9 build/${DOCDIR}/changelog.Debian
- echo "git clone git://git.proxmox.com/git/pve-storage.git\\ngit checkout ${GITVERSION}" > build/${DOCDIR}/SOURCE
- dpkg-deb --build build
- mv build.deb ${DEB}
- rm -rf debian
+ rsync -a src/ build
+ rsync -a debian/ build/debian
+ cp changelog.Debian build/debian/changelog
+ echo "git clone git://git.proxmox.com/git/pve-ha-manager.git\\ngit checkout ${GITVERSION}" > build/debian/SOURCE
+ cd build; dpkg-buildpackage -rfakeroot -b -us -uc
lintian ${DEB}
-
-.PHONY: test
-test:
- make -C test test
-
.PHONY: clean
-clean:
- make -C test clean
- rm -rf build *.deb ${PACKAGE}-*.tar.gz dist *.1.pod *.1.gz
+clean:
+ rm -rf build *.deb ${PACKAGE}-*.tar.gz *.changes
find . -name '*~' -exec rm {} ';'
.PHONY: distclean
+++ /dev/null
-package PVE::HA::CRM;
-
-# Cluster Resource Manager
-
-use strict;
-use warnings;
-
-use PVE::SafeSyslog;
-use PVE::Tools;
-use PVE::HA::Tools;
-
-use PVE::HA::Manager;
-
-# Server can have several state:
-
-my $valid_states = {
- wait_for_quorum => "cluster is not quorate, waiting",
- master => "quorate, and we got the ha_manager lock",
- lost_manager_lock => "we lost the ha_manager lock (watchgog active)",
- slave => "quorate, but we do not own the ha_manager lock",
-};
-
-sub new {
- my ($this, $haenv) = @_;
-
- my $class = ref($this) || $this;
-
- my $self = bless {
- haenv => $haenv,
- manager => undef,
- status => { state => 'startup' },
- }, $class;
-
- $self->set_local_status({ state => 'wait_for_quorum' });
-
- return $self;
-}
-
-sub shutdown_request {
- my ($self) = @_;
-
- syslog('info' , "server received shutdown request")
- if !$self->{shutdown_request};
-
- $self->{shutdown_request} = 1;
-}
-
-sub get_local_status {
- my ($self) = @_;
-
- return $self->{status};
-}
-
-sub set_local_status {
- my ($self, $new) = @_;
-
- die "invalid state '$new->{state}'" if !$valid_states->{$new->{state}};
-
- my $haenv = $self->{haenv};
-
- my $old = $self->{status};
-
- # important: only update if if really changed
- return if $old->{state} eq $new->{state};
-
- $haenv->log('info', "status change $old->{state} => $new->{state}");
-
- $new->{state_change_time} = $haenv->get_time();
-
- $self->{status} = $new;
-
- # fixme: do not use extra class
- if ($new->{state} eq 'master') {
- $self->{manager} = PVE::HA::Manager->new($haenv);
- } else {
- if ($self->{manager}) {
- # fixme: what should we do here?
- $self->{manager}->cleanup();
- $self->{manager} = undef;
- }
- }
-}
-
-sub get_protected_ha_manager_lock {
- my ($self) = @_;
-
- my $haenv = $self->{haenv};
-
- my $count = 0;
- my $starttime = $haenv->get_time();
-
- for (;;) {
-
- if ($haenv->get_ha_manager_lock()) {
- if ($self->{ha_manager_wd}) {
- $haenv->watchdog_update($self->{ha_manager_wd});
- } else {
- my $wfh = $haenv->watchdog_open();
- $self->{ha_manager_wd} = $wfh;
- }
- return 1;
- }
-
- last if ++$count > 5; # try max 5 time
-
- my $delay = $haenv->get_time() - $starttime;
- last if $delay > 5; # for max 5 seconds
-
- $haenv->sleep(1);
- }
-
- return 0;
-}
-
-sub do_one_iteration {
- my ($self) = @_;
-
- my $haenv = $self->{haenv};
-
- my $status = $self->get_local_status();
- my $state = $status->{state};
-
- # do state changes first
-
- if ($state eq 'wait_for_quorum') {
-
- if ($haenv->quorate()) {
- if ($self->get_protected_ha_manager_lock()) {
- $self->set_local_status({ state => 'master' });
- } else {
- $self->set_local_status({ state => 'slave' });
- }
- }
-
- } elsif ($state eq 'slave') {
-
- if ($haenv->quorate()) {
- if ($self->get_protected_ha_manager_lock()) {
- $self->set_local_status({ state => 'master' });
- }
- } else {
- $self->set_local_status({ state => 'wait_for_quorum' });
- }
-
- } elsif ($state eq 'lost_manager_lock') {
-
- if ($haenv->quorate()) {
- if ($self->get_protected_ha_manager_lock()) {
- $self->set_local_status({ state => 'master' });
- }
- }
-
- } elsif ($state eq 'master') {
-
- if (!$self->get_protected_ha_manager_lock()) {
- $self->set_local_status({ state => 'lost_manager_lock'});
- }
- }
-
- $status = $self->get_local_status();
- $state = $status->{state};
-
- # do work
-
- if ($state eq 'wait_for_quorum') {
-
- return 0 if $self->{shutdown_request};
-
- $haenv->sleep(5);
-
- } elsif ($state eq 'master') {
-
- my $manager = $self->{manager};
-
- die "no manager" if !defined($manager);
-
- my $startime = $haenv->get_time();
-
- my $max_time = 10;
-
- # do work (max_time seconds)
- eval {
- # fixme: set alert timer
- $manager->manage();
- };
- if (my $err = $@) {
- $haenv->log('err', "got unexpected error - $err");
- }
-
- $haenv->sleep_until($startime + $max_time);
-
- } elsif ($state eq 'lost_manager_lock') {
-
- if ($self->{ha_manager_wd}) {
- $haenv->watchdog_close($self->{ha_manager_wd});
- delete $self->{ha_manager_wd};
- }
-
- return 0 if $self->{shutdown_request};
-
- $self->set_local_status({ state => 'wait_for_quorum' });
-
- } elsif ($state eq 'slave') {
-
- return 0 if $self->{shutdown_request};
-
- # wait until we get master
-
- } else {
-
- die "got unexpected status '$state'\n";
- }
-
- return 1;
-}
-
-1;
+++ /dev/null
-package PVE::HA::Env;
-
-use strict;
-use warnings;
-
-use PVE::SafeSyslog;
-use PVE::Tools;
-
-# abstract out the cluster environment for a single node
-
-sub new {
- my ($this, $baseclass, $node, @args) = @_;
-
- my $class = ref($this) || $this;
-
- my $plug = $baseclass->new($node, @args);
-
- my $self = bless { plug => $plug }, $class;
-
- return $self;
-}
-
-sub nodename {
- my ($self) = @_;
-
- return $self->{plug}->nodename();
-}
-
-# manager status is stored on cluster, protected by ha_manager_lock
-sub read_manager_status {
- my ($self) = @_;
-
- return $self->{plug}->read_manager_status();
-}
-
-sub write_manager_status {
- my ($self, $status_obj) = @_;
-
- return $self->{plug}->write_manager_status($status_obj);
-}
-
-# lrm status is written by LRM, protected by ha_agent_lock,
-# but can be read by any node (CRM)
-
-sub read_lrm_status {
- my ($self, $node) = @_;
-
- return $self->{plug}->read_lrm_status($node);
-}
-
-sub write_lrm_status {
- my ($self, $status_obj) = @_;
-
- return $self->{plug}->write_lrm_status($status_obj);
-}
-
-# we use this to enable/disbale ha
-sub manager_status_exists {
- my ($self) = @_;
-
- return $self->{plug}->manager_status_exists();
-}
-
-# implement a way to send commands to the CRM master
-sub queue_crm_commands {
- my ($self, $cmd) = @_;
-
- return $self->{plug}->queue_crm_commands($cmd);
-}
-
-sub read_crm_commands {
- my ($self) = @_;
-
- return $self->{plug}->read_crm_commands();
-}
-
-sub read_service_config {
- my ($self) = @_;
-
- return $self->{plug}->read_service_config();
-}
-
-sub change_service_location {
- my ($self, $sid, $node) = @_;
-
- return $self->{plug}->change_service_location($sid, $node);
-}
-
-sub read_group_config {
- my ($self) = @_;
-
- return $self->{plug}->read_group_config();
-}
-
-# this should return a hash containing info
-# what nodes are members and online.
-sub get_node_info {
- my ($self) = @_;
-
- return $self->{plug}->get_node_info();
-}
-
-sub log {
- my ($self, $level, @args) = @_;
-
- return $self->{plug}->log($level, @args);
-}
-
-# aquire a cluster wide manager lock
-sub get_ha_manager_lock {
- my ($self) = @_;
-
- return $self->{plug}->get_ha_manager_lock();
-}
-
-# aquire a cluster wide node agent lock
-sub get_ha_agent_lock {
- my ($self) = @_;
-
- return $self->{plug}->get_ha_agent_lock();
-}
-
-# same as get_ha_agent_lock(), but immeditaley release the lock on success
-sub test_ha_agent_lock {
- my ($self, $node) = @_;
-
- return $self->{plug}->test_ha_agent_lock($node);
-}
-
-# return true when cluster is quorate
-sub quorate {
- my ($self) = @_;
-
- return $self->{plug}->quorate();
-}
-
-# return current time
-# overwrite that if you want to simulate
-sub get_time {
- my ($self) = @_;
-
- return $self->{plug}->get_time();
-}
-
-sub sleep {
- my ($self, $delay) = @_;
-
- return $self->{plug}->sleep($delay);
-}
-
-sub sleep_until {
- my ($self, $end_time) = @_;
-
- return $self->{plug}->sleep_until($end_time);
-}
-
-sub loop_start_hook {
- my ($self, @args) = @_;
-
- return $self->{plug}->loop_start_hook(@args);
-}
-
-sub loop_end_hook {
- my ($self, @args) = @_;
-
- return $self->{plug}->loop_end_hook(@args);
-}
-
-sub watchdog_open {
- my ($self) = @_;
-
- # Note: when using /dev/watchdog, make sure perl does not close
- # the handle automatically at exit!!
-
- return $self->{plug}->watchdog_open();
-}
-
-sub watchdog_update {
- my ($self, $wfh) = @_;
-
- return $self->{plug}->watchdog_update($wfh);
-}
-
-sub watchdog_close {
- my ($self, $wfh) = @_;
-
- return $self->{plug}->watchdog_close($wfh);
-}
-
-sub exec_resource_agent {
- my ($self, $sid, $cmd, @params) = @_;
-
- return $self->{plug}->exec_resource_agent($sid, $cmd, @params)
-}
-
-1;
+++ /dev/null
-
-
-.PHONY: install
-install:
- install -D -m 0644 PVE2.pm ${DESTDIR}${PERLDIR}/PVE/HA/Env/PVE2.pm
+++ /dev/null
-package PVE::HA::Env::PVE2;
-
-use strict;
-use warnings;
-use POSIX qw(:errno_h :fcntl_h);
-use IO::File;
-
-use PVE::SafeSyslog;
-use PVE::Tools;
-use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_lock_file);
-
-use PVE::HA::Tools;
-use PVE::HA::Env;
-use PVE::HA::Groups;
-
-my $lockdir = "/etc/pve/priv/lock";
-
-my $manager_status_filename = "/etc/pve/manager_status";
-my $ha_groups_config = "ha/groups.cfg";
-
-#cfs_register_file($ha_groups_config,
-# sub { PVE::HA::Groups->parse_config(@_); },
-# sub { PVE::HA::Groups->write_config(@_); });
-
-sub new {
- my ($this, $nodename) = @_;
-
- die "missing nodename" if !$nodename;
-
- my $class = ref($this) || $this;
-
- my $self = bless {}, $class;
-
- $self->{nodename} = $nodename;
-
- return $self;
-}
-
-sub nodename {
- my ($self) = @_;
-
- return $self->{nodename};
-}
-
-sub read_manager_status {
- my ($self) = @_;
-
- my $filename = $manager_status_filename;
-
- return PVE::HA::Tools::read_json_from_file($filename, {});
-}
-
-sub write_manager_status {
- my ($self, $status_obj) = @_;
-
- my $filename = $manager_status_filename;
-
- PVE::HA::Tools::write_json_to_file($filename, $status_obj);
-}
-
-sub read_lrm_status {
- my ($self, $node) = @_;
-
- $node = $self->{nodename} if !defined($node);
-
- my $filename = "/etc/pve/nodes/$node/lrm_status";
-
- return PVE::HA::Tools::read_json_from_file($filename, {});
-}
-
-sub write_lrm_status {
- my ($self, $status_obj) = @_;
-
- my $node = $self->{nodename};
-
- my $filename = "/etc/pve/nodes/$node/lrm_status";
-
- PVE::HA::Tools::write_json_to_file($filename, $status_obj);
-}
-
-sub manager_status_exists {
- my ($self) = @_;
-
- return -f $manager_status_filename ? 1 : 0;
-}
-
-sub read_service_config {
- my ($self) = @_;
-
- die "implement me";
-}
-
-sub change_service_location {
- my ($self, $sid, $node) = @_;
-
- die "implement me";
-}
-
-sub read_group_config {
- my ($self) = @_;
-
- return cfs_read_file($ha_groups_config);
-}
-
-sub queue_crm_commands {
- my ($self, $cmd) = @_;
-
- die "implement me";
-}
-
-sub read_crm_commands {
- my ($self) = @_;
-
- die "implement me";
-}
-
-# this should return a hash containing info
-# what nodes are members and online.
-sub get_node_info {
- my ($self) = @_;
-
- die "implement me";
-}
-
-sub log {
- my ($self, $level, $msg) = @_;
-
- chomp $msg;
-
- syslog($level, $msg);
-}
-
-my $last_lock_status = {};
-
-sub get_pve_lock {
- my ($self, $lockid) = @_;
-
- my $got_lock = 0;
-
- my $filename = "$lockdir/$lockid";
-
- my $last = $last_lock_status->{$lockid} || 0;
-
- my $ctime = time();
-
- eval {
-
- mkdir $lockdir;
-
- # pve cluster filesystem not online
- die "can't create '$lockdir' (pmxcfs not mounted?)\n" if ! -d $lockdir;
-
- if ($last && (($ctime - $last) < 100)) { # fixme: what timeout
- utime(0, $ctime, $filename) || # cfs lock update request
- die "cfs lock update failed - $!\n";
- } else {
-
- # fixme: wait some time?
- if (!(mkdir $filename)) {
- utime 0, 0, $filename; # cfs unlock request
- die "can't get cfs lock\n";
- }
- }
-
- $got_lock = 1;
- };
-
- my $err = $@;
-
- $last_lock_status->{$lockid} = $got_lock ? $ctime : 0;
-
- if ($got_lock != $last) {
- if ($got_lock) {
- $self->log('info', "successfully aquired lock '$lockid'");
- } else {
- my $msg = "lost lock '$lockid";
- $msg .= " - $err" if $err;
- $self->log('err', $msg);
- }
- }
-
- return $got_lock;
-}
-
-sub get_ha_manager_lock {
- my ($self) = @_;
-
- my $lockid = "ha_manager_lock";
-
- my $filename = "$lockdir/$lockid";
-
- return $self->get_pve_lock("ha_manager_lock");
-}
-
-sub get_ha_agent_lock {
- my ($self) = @_;
-
- my $node = $self->nodename();
-
- return $self->get_pve_lock("ha_agent_${node}_lock");
-}
-
-sub test_ha_agent_lock {
- my ($self, $node) = @_;
-
- my $lockid = "ha_agent_${node}_lock";
- my $filename = "$lockdir/$lockid";
- my $res = $self->get_pve_lock($lockid);
- rmdir $filename if $res; # cfs unlock
-
- return $res;
-}
-
-sub quorate {
- my ($self) = @_;
-
- my $quorate = 0;
- eval {
- $quorate = PVE::Cluster::check_cfs_quorum();
- };
-
- return $quorate;
-}
-
-sub get_time {
- my ($self) = @_;
-
- return time();
-}
-
-sub sleep {
- my ($self, $delay) = @_;
-
- CORE::sleep($delay);
-}
-
-sub sleep_until {
- my ($self, $end_time) = @_;
-
- for (;;) {
- my $cur_time = time();
-
- last if $cur_time >= $end_time;
-
- $self->sleep(1);
- }
-}
-
-sub loop_start_hook {
- my ($self) = @_;
-
- PVE::Cluster::cfs_update();
-
- $self->{loop_start} = $self->get_time();
-}
-
-sub loop_end_hook {
- my ($self) = @_;
-
- my $delay = $self->get_time() - $self->{loop_start};
-
- warn "loop take too long ($delay seconds)\n" if $delay > 30;
-}
-
-my $watchdog_fh;
-
-my $WDIOC_GETSUPPORT = 0x80285700;
-my $WDIOC_KEEPALIVE = 0x80045705;
-my $WDIOC_SETTIMEOUT = 0xc0045706;
-my $WDIOC_GETTIMEOUT = 0x80045707;
-
-sub watchdog_open {
- my ($self) = @_;
-
- system("modprobe -q softdog soft_noboot=1") if ! -e "/dev/watchdog";
-
- die "watchdog already open\n" if defined($watchdog_fh);
-
- $watchdog_fh = IO::File->new(">/dev/watchdog") ||
- die "unable to open watchdog device - $!\n";
-
- eval {
- my $timeoutbuf = pack('I', 100);
- my $res = ioctl($watchdog_fh, $WDIOC_SETTIMEOUT, $timeoutbuf) ||
- die "unable to set watchdog timeout - $!\n";
- my $timeout = unpack("I", $timeoutbuf);
- die "got wrong watchdog timeout '$timeout'\n" if $timeout != 100;
-
- my $wdinfo = "\x00" x 40;
- $res = ioctl($watchdog_fh, $WDIOC_GETSUPPORT, $wdinfo) ||
- die "unable to get watchdog info - $!\n";
-
- my ($options, $firmware_version, $indentity) = unpack("lla32", $wdinfo);
- die "watchdog does not support magic close\n" if !($options & 0x0100);
-
- };
- if (my $err = $@) {
- $self->watchdog_close();
- die $err;
- }
-
- # fixme: use ioctl to setup watchdog timeout (requires C interface)
-
- $self->log('info', "watchdog active");
-}
-
-sub watchdog_update {
- my ($self, $wfh) = @_;
-
- my $res = $watchdog_fh->syswrite("\0", 1);
- if (!defined($res)) {
- $self->log('err', "watchdog update failed - $!\n");
- return 0;
- }
- if ($res != 1) {
- $self->log('err', "watchdog update failed - write $res bytes\n");
- return 0;
- }
-
- return 1;
-}
-
-sub watchdog_close {
- my ($self, $wfh) = @_;
-
- $watchdog_fh->syswrite("V", 1); # magic watchdog close
- if (!$watchdog_fh->close()) {
- $self->log('err', "watchdog close failed - $!");
- } else {
- $watchdog_fh = undef;
- $self->log('info', "watchdog closed (disabled)");
- }
-}
-
-sub exec_resource_agent {
- my ($self, $sid, $cmd, @params) = @_;
-
- die "implement me";
-}
-
-1;
+++ /dev/null
-package PVE::HA::Groups;
-
-use strict;
-use warnings;
-
-use Data::Dumper;
-use PVE::JSONSchema qw(get_standard_option);
-use PVE::SectionConfig;
-
-use base qw(PVE::SectionConfig);
-
-PVE::JSONSchema::register_format('pve-ha-group-node', \&pve_verify_ha_group_node);
-sub pve_verify_ha_group_node {
- my ($node, $noerr) = @_;
-
- if ($node !~ m/^([a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?)(:\d+)?$/) {
- return undef if $noerr;
- die "value does not look like a valid ha group node\n";
- }
- return $node;
-}
-
-PVE::JSONSchema::register_standard_option('pve-ha-group-node-list', {
- description => "List of cluster node names with optional priority. We use priority '0' as default. The CRM tries to run services on the node with higest priority (also see option 'nofailback').",
- type => 'string', format => 'pve-ha-group-node-list',
- typetext => '<node>[:<pri>]{,<node>[:<pri>]}*',
-});
-
-PVE::JSONSchema::register_standard_option('pve-ha-group-id', {
- description => "The HA group identifier.",
- type => 'string', format => 'pve-configid',
-});
-
-my $defaultData = {
- propertyList => {
- type => { description => "Section type." },
- group => get_standard_option('pve-ha-group-id'),
- nodes => get_standard_option('pve-ha-group-node-list'),
- restricted => {
- description => "Services on unrestricted groups may run on any cluster members if all group members are offline. But they will migrate back as soon as a group member comes online. One can implement a 'preferred node' behavior using an unrestricted group with one member.",
- type => 'boolean',
- optional => 1,
- default => 0,
- },
- nofailback => {
- description => "The CRM tries to run services on the node with the highest priority. If a node with higher priority comes online, the CRM migrates the service to that node. Enabling nofailback prevents that behavior.",
- type => 'boolean',
- optional => 1,
- default => 0,
- },
- comment => {
- description => "Description.",
- type => 'string',
- optional => 1,
- maxLength => 4096,
- },
- },
-};
-
-sub type {
- return 'group';
-}
-
-sub options {
- return {
- nodes => {},
- comment => { optional => 1 },
- };
-}
-
-sub private {
- return $defaultData;
-}
-
-sub parse_section_header {
- my ($class, $line) = @_;
-
- if ($line =~ m/^(\S+):\s*(\S+)\s*$/) {
- my ($type, $group) = (lc($1), $2);
- my $errmsg = undef; # set if you want to skip whole section
- eval { PVE::JSONSchema::pve_verify_configid($group); };
- $errmsg = $@ if $@;
- my $config = {}; # to return additional attributes
- return ($type, $group, $errmsg, $config);
- }
- return undef;
-}
-
-1;
+++ /dev/null
-package PVE::HA::LRM;
-
-# Local Resource Manager
-
-use strict;
-use warnings;
-use Data::Dumper;
-use POSIX qw(:sys_wait_h);
-
-use PVE::SafeSyslog;
-use PVE::Tools;
-use PVE::HA::Tools;
-
-# Server can have several states:
-
-my $valid_states = {
- wait_for_agent_lock => "waiting for agnet lock",
- active => "got agent_lock",
- lost_agent_lock => "lost agent_lock",
-};
-
-sub new {
- my ($this, $haenv) = @_;
-
- my $class = ref($this) || $this;
-
- my $self = bless {
- haenv => $haenv,
- status => { state => 'startup' },
- workers => {},
- results => {},
- }, $class;
-
- $self->set_local_status({ state => 'wait_for_agent_lock' });
-
- return $self;
-}
-
-sub shutdown_request {
- my ($self) = @_;
-
- $self->{shutdown_request} = 1;
-}
-
-sub get_local_status {
- my ($self) = @_;
-
- return $self->{status};
-}
-
-sub set_local_status {
- my ($self, $new) = @_;
-
- die "invalid state '$new->{state}'" if !$valid_states->{$new->{state}};
-
- my $haenv = $self->{haenv};
-
- my $old = $self->{status};
-
- # important: only update if if really changed
- return if $old->{state} eq $new->{state};
-
- $haenv->log('info', "status change $old->{state} => $new->{state}");
-
- $new->{state_change_time} = $haenv->get_time();
-
- $self->{status} = $new;
-}
-
-sub get_protected_ha_agent_lock {
- my ($self) = @_;
-
- my $haenv = $self->{haenv};
-
- my $count = 0;
- my $starttime = $haenv->get_time();
-
- for (;;) {
-
- if ($haenv->get_ha_agent_lock()) {
- if ($self->{ha_agent_wd}) {
- $haenv->watchdog_update($self->{ha_agent_wd});
- } else {
- my $wfh = $haenv->watchdog_open();
- $self->{ha_agent_wd} = $wfh;
- }
- return 1;
- }
-
- last if ++$count > 5; # try max 5 time
-
- my $delay = $haenv->get_time() - $starttime;
- last if $delay > 5; # for max 5 seconds
-
- $haenv->sleep(1);
- }
-
- return 0;
-}
-
-sub do_one_iteration {
- my ($self) = @_;
-
- my $haenv = $self->{haenv};
-
- my $status = $self->get_local_status();
- my $state = $status->{state};
-
- # do state changes first
-
- my $ctime = $haenv->get_time();
-
- if ($state eq 'wait_for_agent_lock') {
-
- my $service_count = 1; # todo: correctly compute
-
- if ($service_count && $haenv->quorate()) {
- if ($self->get_protected_ha_agent_lock()) {
- $self->set_local_status({ state => 'active' });
- }
- }
-
- } elsif ($state eq 'lost_agent_lock') {
-
- if ($haenv->quorate()) {
- if ($self->get_protected_ha_agent_lock()) {
- $self->set_local_status({ state => 'active' });
- }
- }
-
- } elsif ($state eq 'active') {
-
- if (!$self->get_protected_ha_agent_lock()) {
- $self->set_local_status({ state => 'lost_agent_lock'});
- }
- }
-
- $status = $self->get_local_status();
- $state = $status->{state};
-
- # do work
-
- $self->{service_status} = {};
-
- if ($state eq 'wait_for_agent_lock') {
-
- return 0 if $self->{shutdown_request};
-
- $haenv->sleep(5);
-
- } elsif ($state eq 'active') {
-
- my $startime = $haenv->get_time();
-
- my $max_time = 10;
-
- my $shutdown = 0;
-
- # do work (max_time seconds)
- eval {
- # fixme: set alert timer
-
- if ($self->{shutdown_request}) {
-
- # fixme: request service stop or relocate ?
-
- my $service_count = 0; # fixme
-
- if ($service_count == 0) {
-
- if ($self->{ha_agent_wd}) {
- $haenv->watchdog_close($self->{ha_agent_wd});
- delete $self->{ha_agent_wd};
- }
-
- $shutdown = 1;
- }
- } else {
- my $ms = $haenv->read_manager_status();
-
- $self->{service_status} = $ms->{service_status} || {};
-
- $self->manage_resources();
- }
- };
- if (my $err = $@) {
- $haenv->log('err', "got unexpected error - $err");
- }
-
- return 0 if $shutdown;
-
- $haenv->sleep_until($startime + $max_time);
-
- } elsif ($state eq 'lost_agent_lock') {
-
- # Note: watchdog is active an will triger soon!
-
- # so we hope to get the lock back soon!
-
- if ($self->{shutdown_request}) {
-
- my $running_services = 0; # fixme: correctly compute
-
- if ($running_services > 0) {
- $haenv->log('err', "get shutdown request in state 'lost_agent_lock' - " .
- "killing running services");
-
- # fixme: kill all services as fast as possible
- }
-
- # now all services are stopped, so we can close the watchdog
-
- if ($self->{ha_agent_wd}) {
- $haenv->watchdog_close($self->{ha_agent_wd});
- delete $self->{ha_agent_wd};
- }
-
- return 0;
- }
-
- } else {
-
- die "got unexpected status '$state'\n";
-
- }
-
- return 1;
-}
-
-sub manage_resources {
- my ($self) = @_;
-
- my $haenv = $self->{haenv};
-
- my $nodename = $haenv->nodename();
-
- my $ms = $haenv->read_manager_status();
-
- my $ss = $self->{service_status};
-
- foreach my $sid (keys %$ss) {
- my $sd = $ss->{$sid};
- next if !$sd->{node};
- next if !$sd->{uid};
- next if $sd->{node} ne $nodename;
- my $req_state = $sd->{state};
- next if !defined($req_state);
-
- eval {
- $self->queue_resource_command($sid, $sd->{uid}, $req_state, $sd->{target});
- };
- if (my $err = $@) {
- warn "unable to run resource agent for '$sid' - $err"; # fixme
- }
- }
-
- my $starttime = time();
-
- # start workers
- my $max_workers = 4;
-
- while ((time() - $starttime) < 5) {
- my $count = $self->check_active_workers();
-
- foreach my $sid (keys %{$self->{workers}}) {
- last if $count >= $max_workers;
- my $w = $self->{workers}->{$sid};
- if (!$w->{pid}) {
- my $pid = fork();
- if (!defined($pid)) {
- warn "fork worker failed\n";
- $count = 0; last; # abort, try later
- } elsif ($pid == 0) {
- # do work
- my $res = -1;
- eval {
- $res = $haenv->exec_resource_agent($sid, $w->{state}, $w->{target});
- };
- if (my $err = $@) {
- warn $err;
- POSIX::_exit(-1);
- }
- POSIX::_exit($res);
- } else {
- $count++;
- $w->{pid} = $pid;
- }
- }
- }
-
- last if !$count;
-
- sleep(1);
- }
-}
-
-# fixme: use a queue an limit number of parallel workers?
-sub queue_resource_command {
- my ($self, $sid, $uid, $state, $target) = @_;
-
- if (my $w = $self->{workers}->{$sid}) {
- return if $w->{pid}; # already started
- # else, delete and overwrite queue entry with new command
- delete $self->{workers}->{$sid};
- }
-
- $self->{workers}->{$sid} = {
- sid => $sid,
- uid => $uid,
- state => $state,
- };
-
- $self->{workers}->{$sid}->{target} = $target if $target;
-}
-
-sub check_active_workers {
- my ($self) = @_;
-
- # finish/count workers
- my $count = 0;
- foreach my $sid (keys %{$self->{workers}}) {
- my $w = $self->{workers}->{$sid};
- if (my $pid = $w->{pid}) {
- # check status
- my $waitpid = waitpid($pid, WNOHANG);
- if (defined($waitpid) && ($waitpid == $pid)) {
- $self->resource_command_finished($sid, $w->{uid}, $?);
- } else {
- $count++;
- }
- }
- }
-
- return $count;
-}
-
-sub resource_command_finished {
- my ($self, $sid, $uid, $status) = @_;
-
- my $haenv = $self->{haenv};
-
- my $w = delete $self->{workers}->{$sid};
- return if !$w; # should not happen
-
- my $exit_code = -1;
-
- if ($status == -1) {
- $haenv->log('err', "resource agent $sid finished - failed to execute");
- } elsif (my $sig = ($status & 127)) {
- $haenv->log('err', "resource agent $sid finished - got signal $sig");
- } else {
- $exit_code = ($status >> 8);
- }
-
- $self->{results}->{$uid} = {
- sid => $w->{sid},
- state => $w->{state},
- exit_code => $exit_code,
- };
-
- my $ss = $self->{service_status};
-
- # compute hash of valid/existing uids
- my $valid_uids = {};
- foreach my $sid (keys %$ss) {
- my $sd = $ss->{$sid};
- next if !$sd->{uid};
- $valid_uids->{$sd->{uid}} = 1;
- }
-
- my $results = {};
- foreach my $id (keys %{$self->{results}}) {
- next if !$valid_uids->{$id};
- $results->{$id} = $self->{results}->{$id};
- }
- $self->{results} = $results;
-
- $haenv->write_lrm_status($results);
-}
-
-1;
+++ /dev/null
-SOURCES=CRM.pm Env.pm Groups.pm LRM.pm Manager.pm NodeStatus.pm Tools.pm
-
-.PHONY: install
-install:
- install -d -m 0755 ${DESTDIR}${PERLDIR}/PVE/HA
- for i in ${SOURCES}; do install -D -m 0644 $$i ${DESTDIR}${PERLDIR}/PVE/HA/$$i; done
- make -C Env install
-
-.PHONY: installsim
-installsim:
- install -d -m 0755 ${DESTDIR}${PERLDIR}/PVE/HA
- for i in ${SOURCES}; do install -D -m 0644 $$i ${DESTDIR}${PERLDIR}/PVE/HA/$$i; done
- make -C Sim installsim
+++ /dev/null
-package PVE::HA::Manager;
-
-use strict;
-use warnings;
-use Digest::MD5 qw(md5_base64);
-
-use Data::Dumper;
-
-use PVE::HA::NodeStatus;
-
-sub new {
- my ($this, $haenv) = @_;
-
- my $class = ref($this) || $this;
-
- my $ms = $haenv->read_manager_status();
-
- $ms->{master_node} = $haenv->nodename();
-
- my $ns = PVE::HA::NodeStatus->new($haenv, $ms->{node_status} || {});
-
- # fixme: use separate class PVE::HA::ServiceStatus
- my $ss = $ms->{service_status} || {};
-
- my $self = bless {
- haenv => $haenv,
- ms => $ms, # master status
- ns => $ns, # PVE::HA::NodeStatus
- ss => $ss, # service status
- }, $class;
-
- return $self;
-}
-
-sub cleanup {
- my ($self) = @_;
-
- # todo: ?
-}
-
-sub flush_master_status {
- my ($self) = @_;
-
- my ($haenv, $ms, $ns, $ss) = ($self->{haenv}, $self->{ms}, $self->{ns}, $self->{ss});
-
- $ms->{node_status} = $ns->{status};
- $ms->{service_status} = $ss;
-
- $haenv->write_manager_status($ms);
-}
-
-sub select_service_node {
- my ($groups, $online_node_usage, $service_conf, $current_node, $try_next) = @_;
-
- my $group = { 'nodes' => $service_conf->{node} }; # default group
-
- $group = $groups->{ids}->{$service_conf->{group}} if $service_conf->{group} &&
- $groups->{ids}->{$service_conf->{group}};
-
- my $pri_groups = {};
- my $group_members = {};
- foreach my $entry (PVE::Tools::split_list($group->{nodes})) {
- my ($node, $pri) = ($entry, 0);
- if ($entry =~ m/^(\S+):(\d+)$/) {
- ($node, $pri) = ($1, $2);
- }
- next if !defined($online_node_usage->{$node}); # offline
- $pri_groups->{$pri}->{$node} = 1;
- $group_members->{$node} = $pri;
- }
-
-
- # add non-group members to unrestricted groups (priority -1)
- if (!$group->{restricted}) {
- my $pri = -1;
- foreach my $node (keys %$online_node_usage) {
- next if defined($group_members->{$node});
- $pri_groups->{$pri}->{$node} = 1;
- $group_members->{$node} = -1;
- }
- }
-
-
- my @pri_list = sort {$b <=> $a} keys %$pri_groups;
- return undef if !scalar(@pri_list);
-
- if (!$try_next && $group->{nofailback} && defined($group_members->{$current_node})) {
- return $current_node;
- }
-
- # select node from top priority node list
-
- my $top_pri = $pri_list[0];
-
- my @nodes = sort { $online_node_usage->{$a} <=> $online_node_usage->{$b} } keys %{$pri_groups->{$top_pri}};
-
- my $found;
- for (my $i = scalar(@nodes) - 1; $i >= 0; $i--) {
- my $node = $nodes[$i];
- if ($node eq $current_node) {
- $found = $i;
- last;
- }
- }
-
- if ($try_next) {
-
- if (defined($found) && ($found < (scalar(@nodes) - 1))) {
- return $nodes[$found + 1];
- } else {
- return $nodes[0];
- }
-
- } else {
-
- return $nodes[$found] if defined($found);
-
- return $nodes[0];
-
- }
-}
-
-my $uid_counter = 0;
-
-my $valid_service_states = {
- stopped => 1,
- request_stop => 1,
- started => 1,
- fence => 1,
- migrate => 1,
- relocate => 1,
- error => 1,
-};
-
-sub recompute_online_node_usage {
- my ($self) = @_;
-
- my $online_node_usage = {};
-
- my $online_nodes = $self->{ns}->list_online_nodes();
-
- foreach my $node (@$online_nodes) {
- $online_node_usage->{$node} = 0;
- }
-
- foreach my $sid (keys %{$self->{ss}}) {
- my $sd = $self->{ss}->{$sid};
- my $state = $sd->{state};
- if (defined($online_node_usage->{$sd->{node}})) {
- if (($state eq 'started') || ($state eq 'request_stop') ||
- ($state eq 'fence') || ($state eq 'error')) {
- $online_node_usage->{$sd->{node}}++;
- } elsif (($state eq 'migrate') || ($state eq 'relocate')) {
- $online_node_usage->{$sd->{target}}++;
- } elsif ($state eq 'stopped') {
- # do nothing
- } else {
- die "should not be reached";
- }
- }
- }
-
- $self->{online_node_usage} = $online_node_usage;
-}
-
-my $change_service_state = sub {
- my ($self, $sid, $new_state, %params) = @_;
-
- my ($haenv, $ss) = ($self->{haenv}, $self->{ss});
-
- my $sd = $ss->{$sid} || die "no such service '$sid";
-
- my $old_state = $sd->{state};
- my $old_node = $sd->{node};
-
- die "no state change" if $old_state eq $new_state; # just to be sure
-
- die "invalid CRM service state '$new_state'\n" if !$valid_service_states->{$new_state};
-
- foreach my $k (keys %$sd) { delete $sd->{$k}; };
-
- $sd->{state} = $new_state;
- $sd->{node} = $old_node;
-
- my $text_state = '';
- foreach my $k (keys %params) {
- my $v = $params{$k};
- $text_state .= ", " if $text_state;
- $text_state .= "$k = $v";
- $sd->{$k} = $v;
- }
-
- $self->recompute_online_node_usage();
-
- $uid_counter++;
- $sd->{uid} = md5_base64($new_state . $$ . time() . $uid_counter);
-
- $text_state = " ($text_state)" if $text_state;
- $haenv->log('info', "service '$sid': state changed from '${old_state}' to '${new_state}' $text_state\n");
-};
-
-# read LRM status for all active nodes
-sub read_lrm_status {
- my ($self) = @_;
-
- my $nodes = $self->{ns}->list_online_nodes();
- my $haenv = $self->{haenv};
-
- my $res = {};
-
- foreach my $node (@$nodes) {
- my $ls = $haenv->read_lrm_status($node);
- foreach my $uid (keys %$ls) {
- next if $res->{$uid}; # should not happen
- $res->{$uid} = $ls->{$uid};
- }
- }
-
- return $res;
-}
-
-# read new crm commands and save them into crm master status
-sub update_crm_commands {
- my ($self) = @_;
-
- my ($haenv, $ms, $ns, $ss) = ($self->{haenv}, $self->{ms}, $self->{ns}, $self->{ss});
-
- my $cmdlist = $haenv->read_crm_commands();
-
- foreach my $cmd (split(/\n/, $cmdlist)) {
- chomp $cmd;
-
- if ($cmd =~ m/^(migrate|relocate)\s+(\S+)\s+(\S+)$/) {
- my ($task, $sid, $node) = ($1, $2, $3);
- if (my $sd = $ss->{$sid}) {
- if (!$ns->node_is_online($node)) {
- $haenv->log('err', "crm command error - node not online: $cmd");
- } else {
- if ($node eq $sd->{node}) {
- $haenv->log('info', "ignore crm command - service already on target node: $cmd");
- } else {
- $haenv->log('info', "got crm command: $cmd");
- $ss->{$sid}->{cmd} = [ $task, $node];
- }
- }
- } else {
- $haenv->log('err', "crm command error - no such service: $cmd");
- }
-
- } else {
- $haenv->log('err', "unable to parse crm command: $cmd");
- }
- }
-
-}
-
-sub manage {
- my ($self) = @_;
-
- my ($haenv, $ms, $ns, $ss) = ($self->{haenv}, $self->{ms}, $self->{ns}, $self->{ss});
-
- $ns->update($haenv->get_node_info());
-
- if (!$ns->node_is_online($haenv->nodename())) {
- $haenv->log('info', "master seems offline\n");
- return;
- }
-
- my $lrm_status = $self->read_lrm_status();
-
- my $sc = $haenv->read_service_config();
-
- $self->{groups} = $haenv->read_group_config(); # update
-
- # compute new service status
-
- # add new service
- foreach my $sid (keys %$sc) {
- next if $ss->{$sid}; # already there
- $haenv->log('info', "Adding new service '$sid'\n");
- # assume we are running to avoid relocate running service at add
- $ss->{$sid} = { state => 'started', node => $sc->{$sid}->{node}};
- }
-
- $self->update_crm_commands();
-
- for (;;) {
- my $repeat = 0;
-
- $self->recompute_online_node_usage();
-
- foreach my $sid (keys %$ss) {
- my $sd = $ss->{$sid};
- my $cd = $sc->{$sid} || { state => 'disabled' };
-
- my $lrm_res = $sd->{uid} ? $lrm_status->{$sd->{uid}} : undef;
-
- my $last_state = $sd->{state};
-
- if ($last_state eq 'stopped') {
-
- $self->next_state_stopped($sid, $cd, $sd, $lrm_res);
-
- } elsif ($last_state eq 'started') {
-
- $self->next_state_started($sid, $cd, $sd, $lrm_res);
-
- } elsif ($last_state eq 'migrate' || $last_state eq 'relocate') {
-
- $self->next_state_migrate_relocate($sid, $cd, $sd, $lrm_res);
-
- } elsif ($last_state eq 'fence') {
-
- # do nothing here - wait until fenced
-
- } elsif ($last_state eq 'request_stop') {
-
- $self->next_state_request_stop($sid, $cd, $sd, $lrm_res);
-
- } elsif ($last_state eq 'error') {
-
- # fixme:
-
- } else {
-
- die "unknown service state '$last_state'";
- }
-
- $repeat = 1 if $sd->{state} ne $last_state;
- }
-
- # handle fencing
- my $fenced_nodes = {};
- foreach my $sid (keys %$ss) {
- my $sd = $ss->{$sid};
- next if $sd->{state} ne 'fence';
-
- if (!defined($fenced_nodes->{$sd->{node}})) {
- $fenced_nodes->{$sd->{node}} = $ns->fence_node($sd->{node}) || 0;
- }
-
- next if !$fenced_nodes->{$sd->{node}};
-
- # node fence was sucessful - mark service as stopped
- &$change_service_state($self, $sid, 'stopped');
- }
-
- last if !$repeat;
- }
-
- # remove stale services
- # fixme:
-
- $self->flush_master_status();
-}
-
-# functions to compute next service states
-# $cd: service configuration data (read only)
-# $sd: service status data (read only)
-#
-# Note: use change_service_state() to alter state
-#
-
-sub next_state_request_stop {
- my ($self, $sid, $cd, $sd, $lrm_res) = @_;
-
- my $haenv = $self->{haenv};
- my $ns = $self->{ns};
-
- # check result from LRM daemon
- if ($lrm_res) {
- my $exit_code = $lrm_res->{exit_code};
- if ($exit_code == 0) {
- &$change_service_state($self, $sid, 'stopped');
- return;
- } else {
- &$change_service_state($self, $sid, 'error'); # fixme: what state?
- return;
- }
- }
-
- if (!$ns->node_is_online($sd->{node})) {
- &$change_service_state($self, $sid, 'fence');
- return;
- }
-}
-
-sub next_state_migrate_relocate {
- my ($self, $sid, $cd, $sd, $lrm_res) = @_;
-
- my $haenv = $self->{haenv};
- my $ns = $self->{ns};
-
- # check result from LRM daemon
- if ($lrm_res) {
- my $exit_code = $lrm_res->{exit_code};
- if ($exit_code == 0) {
- &$change_service_state($self, $sid, 'started', node => $sd->{target});
- return;
- } else {
- $haenv->log('err', "service '$sid' - migration failed (exit code $exit_code)");
- &$change_service_state($self, $sid, 'started', node => $sd->{node});
- return;
- }
- }
-
- if (!$ns->node_is_online($sd->{node})) {
- &$change_service_state($self, $sid, 'fence');
- return;
- }
-}
-
-
-sub next_state_stopped {
- my ($self, $sid, $cd, $sd, $lrm_res) = @_;
-
- my $haenv = $self->{haenv};
- my $ns = $self->{ns};
-
- if ($sd->{node} ne $cd->{node}) {
- # this can happen if we fence a node with active migrations
- # hack: modify $sd (normally this should be considered read-only)
- $haenv->log('info', "fixup service '$sid' location ($sd->{node} => $cd->{node}");
- $sd->{node} = $cd->{node};
- }
-
- if ($sd->{cmd}) {
- my ($cmd, $target) = @{$sd->{cmd}};
- delete $sd->{cmd};
-
- if ($cmd eq 'migrate' || $cmd eq 'relocate') {
- if (!$ns->node_is_online($target)) {
- $haenv->log('err', "ignore service '$sid' $cmd request - node '$target' not online");
- } elsif ($sd->{node} eq $target) {
- $haenv->log('info', "ignore service '$sid' $cmd request - service already on node '$target'");
- } else {
- $haenv->change_service_location($sid, $target);
- $cd->{node} = $sd->{node} = $target; # fixme: $sd is read-only??!!
- $haenv->log('info', "$cmd service '$sid' to node '$target' (stopped)");
- }
- } else {
- $haenv->log('err', "unknown command '$cmd' for service '$sid'");
- }
- }
-
- if ($cd->{state} eq 'disabled') {
- # do nothing
- return;
- }
-
- if ($cd->{state} eq 'enabled') {
- if (my $node = select_service_node($self->{groups}, $self->{online_node_usage}, $cd, $sd->{node})) {
- if ($node && ($sd->{node} ne $node)) {
- $haenv->change_service_location($sid, $node);
- }
- &$change_service_state($self, $sid, 'started', node => $node);
- } else {
- # fixme: warn
- }
-
- return;
- }
-
- $haenv->log('err', "service '$sid' - unknown state '$cd->{state}' in service configuration");
-}
-
-sub next_state_started {
- my ($self, $sid, $cd, $sd, $lrm_res) = @_;
-
- my $haenv = $self->{haenv};
- my $ns = $self->{ns};
-
- if (!$ns->node_is_online($sd->{node})) {
-
- &$change_service_state($self, $sid, 'fence');
- return;
- }
-
- if ($cd->{state} eq 'disabled') {
- &$change_service_state($self, $sid, 'request_stop');
- return;
- }
-
- if ($cd->{state} eq 'enabled') {
-
- if ($sd->{cmd}) {
- my ($cmd, $target) = @{$sd->{cmd}};
- delete $sd->{cmd};
-
- if ($cmd eq 'migrate' || $cmd eq 'relocate') {
- if (!$ns->node_is_online($target)) {
- $haenv->log('err', "ignore service '$sid' $cmd request - node '$target' not online");
- } elsif ($sd->{node} eq $target) {
- $haenv->log('info', "ignore service '$sid' $cmd request - service already on node '$target'");
- } else {
- $haenv->log('info', "$cmd service '$sid' to node '$target' (running)");
- &$change_service_state($self, $sid, $cmd, node => $sd->{node}, target => $target);
- }
- } else {
- $haenv->log('err', "unknown command '$cmd' for service '$sid'");
- }
- } else {
-
- my $try_next = 0;
- if ($lrm_res && ($lrm_res->{exit_code} != 0)) { # fixme: other exit codes?
- $try_next = 1;
- }
-
- my $node = select_service_node($self->{groups}, $self->{online_node_usage},
- $cd, $sd->{node}, $try_next);
-
- if ($node && ($sd->{node} ne $node)) {
- $haenv->log('info', "migrate service '$sid' to node '$node' (running)");
- &$change_service_state($self, $sid, 'migrate', node => $sd->{node}, target => $node);
- } else {
- # do nothing
- }
- }
-
- return;
- }
-
- $haenv->log('err', "service '$sid' - unknown state '$cd->{state}' in service configuration");
-}
-
-1;
+++ /dev/null
-package PVE::HA::NodeStatus;
-
-use strict;
-use warnings;
-
-use Data::Dumper;
-
-sub new {
- my ($this, $haenv, $status) = @_;
-
- my $class = ref($this) || $this;
-
- my $self = bless {
- haenv => $haenv,
- status => $status,
- }, $class;
-
- return $self;
-}
-
-# possible node state:
-my $valid_node_states = {
- online => "node online and member of quorate partition",
- unknown => "not member of quorate partition, but possibly still running",
- fence => "node needs to be fenced",
-};
-
-sub get_node_state {
- my ($self, $node) = @_;
-
- $self->{status}->{$node} = 'unknown'
- if !$self->{status}->{$node};
-
- return $self->{status}->{$node};
-}
-
-sub node_is_online {
- my ($self, $node) = @_;
-
- return $self->get_node_state($node) eq 'online';
-}
-
-sub list_online_nodes {
- my ($self) = @_;
-
- my $res = [];
-
- foreach my $node (sort keys %{$self->{status}}) {
- next if $self->{status}->{$node} ne 'online';
- push @$res, $node;
- }
-
- return $res;
-}
-
-my $set_node_state = sub {
- my ($self, $node, $state) = @_;
-
- my $haenv = $self->{haenv};
-
- die "unknown node state '$state'\n"
- if !defined($valid_node_states->{$state});
-
- my $last_state = $self->get_node_state($node);
-
- return if $state eq $last_state;
-
- $self->{status}->{$node} = $state;
-
- $haenv->log('info', "node '$node': state changed from " .
- "'$last_state' => '$state'\n");
-
-};
-
-sub update {
- my ($self, $node_info) = @_;
-
- foreach my $node (keys %$node_info) {
- my $d = $node_info->{$node};
- next if !$d->{online};
-
- my $state = $self->get_node_state($node);
-
- if ($state eq 'online') {
- # &$set_node_state($self, $node, 'online');
- } elsif ($state eq 'unknown') {
- &$set_node_state($self, $node, 'online');
- } elsif ($state eq 'fence') {
- # do nothing, wait until fenced
- } else {
- die "detected unknown node state '$state";
- }
- }
-
- foreach my $node (keys %{$self->{status}}) {
- my $d = $node_info->{$node};
- next if $d && $d->{online};
-
- my $state = $self->get_node_state($node);
-
- # node is not inside quorate partition, possibly not active
-
- if ($state eq 'online') {
- &$set_node_state($self, $node, 'unknown');
- } elsif ($state eq 'unknown') {
- # &$set_node_state($self, $node, 'unknown');
- } elsif ($state eq 'fence') {
- # do nothing, wait until fenced
- } else {
- die "detected unknown node state '$state";
- }
-
- }
-}
-
-# start fencing
-sub fence_node {
- my ($self, $node) = @_;
-
- my $haenv = $self->{haenv};
-
- my $state = $self->get_node_state($node);
-
- if ($state ne 'fence') {
- &$set_node_state($self, $node, 'fence');
- }
-
- my $success = $haenv->test_ha_agent_lock($node);
-
- if ($success) {
- $haenv->log("info", "fencing: acknowleged - got agent lock for node '$node'");
- &$set_node_state($self, $node, 'unknown');
- }
-
- return $success;
-}
-
-1;
+++ /dev/null
-package PVE::HA::Sim::Env;
-
-use strict;
-use warnings;
-use POSIX qw(strftime EINTR);
-use Data::Dumper;
-use JSON;
-use IO::File;
-use Fcntl qw(:DEFAULT :flock);
-
-use PVE::HA::Tools;
-use PVE::HA::Env;
-
-sub new {
- my ($this, $nodename, $hardware, $log_id) = @_;
-
- die "missing nodename" if !$nodename;
- die "missing log_id" if !$log_id;
-
- my $class = ref($this) || $this;
-
- my $self = bless {}, $class;
-
- $self->{statusdir} = $hardware->statusdir();
- $self->{nodename} = $nodename;
-
- $self->{hardware} = $hardware;
- $self->{lock_timeout} = 120;
-
- $self->{log_id} = $log_id;
-
- return $self;
-}
-
-sub nodename {
- my ($self) = @_;
-
- return $self->{nodename};
-}
-
-sub sim_get_lock {
- my ($self, $lock_name, $unlock) = @_;
-
- return 0 if !$self->quorate();
-
- my $filename = "$self->{statusdir}/cluster_locks";
-
- my $code = sub {
-
- my $data = PVE::HA::Tools::read_json_from_file($filename, {});
-
- my $res;
-
- my $nodename = $self->nodename();
- my $ctime = $self->get_time();
-
- if ($unlock) {
-
- if (my $d = $data->{$lock_name}) {
- my $tdiff = $ctime - $d->{time};
-
- if ($tdiff > $self->{lock_timeout}) {
- $res = 1;
- } elsif (($tdiff <= $self->{lock_timeout}) && ($d->{node} eq $nodename)) {
- delete $data->{$lock_name};
- $res = 1;
- } else {
- $res = 0;
- }
- }
-
- } else {
-
- if (my $d = $data->{$lock_name}) {
-
- my $tdiff = $ctime - $d->{time};
-
- if ($tdiff <= $self->{lock_timeout}) {
- if ($d->{node} eq $nodename) {
- $d->{time} = $ctime;
- $res = 1;
- } else {
- $res = 0;
- }
- } else {
- $self->log('info', "got lock '$lock_name'");
- $d->{node} = $nodename;
- $d->{time} = $ctime;
- $res = 1;
- }
-
- } else {
- $data->{$lock_name} = {
- time => $ctime,
- node => $nodename,
- };
- $self->log('info', "got lock '$lock_name'");
- $res = 1;
- }
- }
-
- PVE::HA::Tools::write_json_to_file($filename, $data);
-
- return $res;
- };
-
- return $self->{hardware}->global_lock($code);
-}
-
-sub read_manager_status {
- my ($self) = @_;
-
- my $filename = "$self->{statusdir}/manager_status";
-
- return PVE::HA::Tools::read_json_from_file($filename, {});
-}
-
-sub write_manager_status {
- my ($self, $status_obj) = @_;
-
- my $filename = "$self->{statusdir}/manager_status";
-
- PVE::HA::Tools::write_json_to_file($filename, $status_obj);
-}
-
-sub manager_status_exists {
- my ($self) = @_;
-
- my $filename = "$self->{statusdir}/manager_status";
-
- return -f $filename ? 1 : 0;
-}
-
-sub read_lrm_status {
- my ($self, $node) = @_;
-
- $node = $self->{nodename} if !defined($node);
-
- return $self->{hardware}->read_lrm_status($node);
-}
-
-sub write_lrm_status {
- my ($self, $status_obj) = @_;
-
- my $node = $self->{nodename};
-
- return $self->{hardware}->write_lrm_status($node, $status_obj);
-}
-
-sub read_service_config {
- my ($self) = @_;
-
- return $self->{hardware}->read_service_config();
-}
-
-sub read_group_config {
- my ($self) = @_;
-
- return $self->{hardware}->read_group_config();
-}
-
-sub change_service_location {
- my ($self, $sid, $node) = @_;
-
- return $self->{hardware}->change_service_location($sid, $node);
-}
-
-sub queue_crm_commands {
- my ($self, $cmd) = @_;
-
- return $self->{hardware}->queue_crm_commands($cmd);
-}
-
-sub read_crm_commands {
- my ($self) = @_;
-
- return $self->{hardware}->read_crm_commands();
-}
-
-sub log {
- my ($self, $level, $msg) = @_;
-
- chomp $msg;
-
- my $time = $self->get_time();
-
- printf("%-5s %5d %12s: $msg\n", $level, $time, "$self->{nodename}/$self->{log_id}");
-}
-
-sub get_time {
- my ($self) = @_;
-
- die "implement in subclass";
-}
-
-sub sleep {
- my ($self, $delay) = @_;
-
- die "implement in subclass";
-}
-
-sub sleep_until {
- my ($self, $end_time) = @_;
-
- die "implement in subclass";
-}
-
-sub get_ha_manager_lock {
- my ($self) = @_;
-
- return $self->sim_get_lock('ha_manager_lock');
-}
-
-sub get_ha_agent_lock_name {
- my ($self, $node) = @_;
-
- $node = $self->nodename() if !$node;
-
- return "ha_agent_${node}_lock";
-}
-
-sub get_ha_agent_lock {
- my ($self) = @_;
-
- my $lck = $self->get_ha_agent_lock_name();
- return $self->sim_get_lock($lck);
-}
-
-sub test_ha_agent_lock {
- my ($self, $node) = @_;
-
- my $lck = $self->get_ha_agent_lock_name($node);
- my $res = $self->sim_get_lock($lck);
- $self->sim_get_lock($lck, 1) if $res; # unlock
- return $res;
-}
-
-# return true when cluster is quorate
-sub quorate {
- my ($self) = @_;
-
- my ($node_info, $quorate) = $self->{hardware}->get_node_info();
- my $node = $self->nodename();
- return 0 if !$node_info->{$node}->{online};
- return $quorate;
-}
-
-sub get_node_info {
- my ($self) = @_;
-
- return $self->{hardware}->get_node_info();
-}
-
-sub loop_start_hook {
- my ($self, $starttime) = @_;
-
- # do nothing, overwrite in subclass
-}
-
-sub loop_end_hook {
- my ($self) = @_;
-
- # do nothing, overwrite in subclass
-}
-
-sub watchdog_open {
- my ($self) = @_;
-
- my $node = $self->nodename();
-
- return $self->{hardware}->watchdog_open($node);
-}
-
-sub watchdog_update {
- my ($self, $wfh) = @_;
-
- return $self->{hardware}->watchdog_update($wfh);
-}
-
-sub watchdog_close {
- my ($self, $wfh) = @_;
-
- return $self->{hardware}->watchdog_close($wfh);
-}
-
-sub exec_resource_agent {
- my ($self, $sid, $cmd, @params) = @_;
-
- die "implement me";
-}
-
-1;
+++ /dev/null
-package PVE::HA::Sim::Hardware;
-
-# Simulate Hardware resources
-
-# power supply for nodes: on/off
-# network connection to nodes: on/off
-# watchdog devices for nodes
-
-use strict;
-use warnings;
-use POSIX qw(strftime EINTR);
-use Data::Dumper;
-use JSON;
-use IO::File;
-use Fcntl qw(:DEFAULT :flock);
-use File::Copy;
-use File::Path qw(make_path remove_tree);
-use PVE::HA::Groups;
-
-PVE::HA::Groups->register();
-PVE::HA::Groups->init();
-
-my $watchdog_timeout = 60;
-
-
-# Status directory layout
-#
-# configuration
-#
-# $testdir/cmdlist Command list for simulation
-# $testdir/hardware_status Hardware description (number of nodes, ...)
-# $testdir/manager_status CRM status (start with {})
-# $testdir/service_config Service configuration
-# $testdir/groups HA groups configuration
-# $testdir/service_status_<node> Service status
-
-#
-# runtime status for simulation system
-#
-# $testdir/status/cluster_locks Cluster locks
-# $testdir/status/hardware_status Hardware status (power/network on/off)
-# $testdir/status/watchdog_status Watchdog status
-#
-# runtime status
-#
-# $testdir/status/lrm_status_<node> LRM status
-# $testdir/status/manager_status CRM status
-# $testdir/status/crm_commands CRM command queue
-# $testdir/status/service_config Service configuration
-# $testdir/status/service_status_<node> Service status
-# $testdir/status/groups HA groups configuration
-
-sub read_lrm_status {
- my ($self, $node) = @_;
-
- my $filename = "$self->{statusdir}/lrm_status_$node";
-
- return PVE::HA::Tools::read_json_from_file($filename, {});
-}
-
-sub write_lrm_status {
- my ($self, $node, $status_obj) = @_;
-
- my $filename = "$self->{statusdir}/lrm_status_$node";
-
- PVE::HA::Tools::write_json_to_file($filename, $status_obj);
-}
-
-sub read_hardware_status_nolock {
- my ($self) = @_;
-
- my $filename = "$self->{statusdir}/hardware_status";
-
- my $raw = PVE::Tools::file_get_contents($filename);
- my $cstatus = decode_json($raw);
-
- return $cstatus;
-}
-
-sub write_hardware_status_nolock {
- my ($self, $cstatus) = @_;
-
- my $filename = "$self->{statusdir}/hardware_status";
-
- PVE::Tools::file_set_contents($filename, encode_json($cstatus));
-};
-
-sub read_service_config {
- my ($self) = @_;
-
- my $filename = "$self->{statusdir}/service_config";
- my $conf = PVE::HA::Tools::read_json_from_file($filename);
-
- foreach my $sid (keys %$conf) {
- my $d = $conf->{$sid};
-
- die "service '$sid' without assigned node!" if !$d->{node};
-
- if ($sid =~ m/^pvevm:(\d+)$/) {
- $d->{type} = 'pvevm';
- $d->{name} = $1;
- } else {
- die "implement me";
- }
- $d->{state} = 'disabled' if !$d->{state};
- }
-
- return $conf;
-}
-
-sub write_service_config {
- my ($self, $conf) = @_;
-
- $self->{service_config} = $conf;
-
- my $filename = "$self->{statusdir}/service_config";
- return PVE::HA::Tools::write_json_to_file($filename, $conf);
-}
-
-sub change_service_location {
- my ($self, $sid, $node) = @_;
-
- my $conf = $self->read_service_config();
-
- die "no such service '$sid'\n" if !$conf->{$sid};
-
- $conf->{$sid}->{node} = $node;
-
- $self->write_service_config($conf);
-}
-
-sub queue_crm_commands {
- my ($self, $cmd) = @_;
-
- chomp $cmd;
-
- my $code = sub {
- my $data = '';
- my $filename = "$self->{statusdir}/crm_commands";
- if (-f $filename) {
- $data = PVE::Tools::file_get_contents($filename);
- }
- $data .= "$cmd\n";
- PVE::Tools::file_set_contents($filename, $data);
- };
-
- $self->global_lock($code);
-
- return undef;
-}
-
-sub read_crm_commands {
- my ($self) = @_;
-
- my $code = sub {
- my $data = '';
-
- my $filename = "$self->{statusdir}/crm_commands";
- if (-f $filename) {
- $data = PVE::Tools::file_get_contents($filename);
- }
- PVE::Tools::file_set_contents($filename, '');
-
- return $data;
- };
-
- return $self->global_lock($code);
-}
-
-sub read_group_config {
- my ($self) = @_;
-
- my $filename = "$self->{statusdir}/groups";
- my $raw = '';
- $raw = PVE::Tools::file_get_contents($filename) if -f $filename;
-
- return PVE::HA::Groups->parse_config($filename, $raw);
-}
-
-sub read_service_status {
- my ($self, $node) = @_;
-
- my $filename = "$self->{statusdir}/service_status_$node";
- return PVE::HA::Tools::read_json_from_file($filename);
-}
-
-sub write_service_status {
- my ($self, $node, $data) = @_;
-
- my $filename = "$self->{statusdir}/service_status_$node";
- my $res = PVE::HA::Tools::write_json_to_file($filename, $data);
-
- # fixme: add test if a service runs on two nodes!!!
-
- return $res;
-}
-
-my $default_group_config = <<__EOD;
-group: prefer_node1
- nodes node1
-
-group: prefer_node2
- nodes node2
-
-group: prefer_node3
- nodes node3
-__EOD
-
-sub new {
- my ($this, $testdir) = @_;
-
- die "missing testdir" if !$testdir;
-
- my $class = ref($this) || $this;
-
- my $self = bless {}, $class;
-
- my $statusdir = $self->{statusdir} = "$testdir/status";
-
- remove_tree($statusdir);
- mkdir $statusdir;
-
- # copy initial configuartion
- copy("$testdir/manager_status", "$statusdir/manager_status"); # optional
-
- if (-f "$testdir/groups") {
- copy("$testdir/groups", "$statusdir/groups");
- } else {
- PVE::Tools::file_set_contents("$statusdir/groups", $default_group_config);
- }
-
- if (-f "$testdir/service_config") {
- copy("$testdir/service_config", "$statusdir/service_config");
- } else {
- my $conf = {
- 'pvevm:101' => { node => 'node1', group => 'prefer_node1' },
- 'pvevm:102' => { node => 'node2', group => 'prefer_node2' },
- 'pvevm:103' => { node => 'node3', group => 'prefer_node3' },
- 'pvevm:104' => { node => 'node1', group => 'prefer_node1' },
- 'pvevm:105' => { node => 'node2', group => 'prefer_node2' },
- 'pvevm:106' => { node => 'node3', group => 'prefer_node3' },
- };
- $self->write_service_config($conf);
- }
-
- if (-f "$testdir/hardware_status") {
- copy("$testdir/hardware_status", "$statusdir/hardware_status") ||
- die "Copy failed: $!\n";
- } else {
- my $cstatus = {
- node1 => { power => 'off', network => 'off' },
- node2 => { power => 'off', network => 'off' },
- node3 => { power => 'off', network => 'off' },
- };
- $self->write_hardware_status_nolock($cstatus);
- }
-
-
- my $cstatus = $self->read_hardware_status_nolock();
-
- foreach my $node (sort keys %$cstatus) {
- $self->{nodes}->{$node} = {};
-
- if (-f "$testdir/service_status_$node") {
- copy("$testdir/service_status_$node", "$statusdir/service_status_$node");
- } else {
- $self->write_service_status($node, {});
- }
- }
-
- $self->{service_config} = $self->read_service_config();
-
- return $self;
-}
-
-sub get_time {
- my ($self) = @_;
-
- die "implement in subclass";
-}
-
-sub log {
- my ($self, $level, $msg, $id) = @_;
-
- chomp $msg;
-
- my $time = $self->get_time();
-
- $id = 'hardware' if !$id;
-
- printf("%-5s %5d %12s: $msg\n", $level, $time, $id);
-}
-
-sub statusdir {
- my ($self, $node) = @_;
-
- return $self->{statusdir};
-}
-
-sub global_lock {
- my ($self, $code, @param) = @_;
-
- my $lockfile = "$self->{statusdir}/hardware.lck";
- my $fh = IO::File->new(">>$lockfile") ||
- die "unable to open '$lockfile'\n";
-
- my $success;
- for (;;) {
- $success = flock($fh, LOCK_EX);
- if ($success || ($! != EINTR)) {
- last;
- }
- if (!$success) {
- close($fh);
- die "can't aquire lock '$lockfile' - $!\n";
- }
- }
-
- my $res;
-
- eval { $res = &$code($fh, @param) };
- my $err = $@;
-
- close($fh);
-
- die $err if $err;
-
- return $res;
-}
-
-my $compute_node_info = sub {
- my ($self, $cstatus) = @_;
-
- my $node_info = {};
-
- my $node_count = 0;
- my $online_count = 0;
-
- foreach my $node (keys %$cstatus) {
- my $d = $cstatus->{$node};
-
- my $online = ($d->{power} eq 'on' && $d->{network} eq 'on') ? 1 : 0;
- $node_info->{$node}->{online} = $online;
-
- $node_count++;
- $online_count++ if $online;
- }
-
- my $quorate = ($online_count > int($node_count/2)) ? 1 : 0;
-
- if (!$quorate) {
- foreach my $node (keys %$cstatus) {
- my $d = $cstatus->{$node};
- $node_info->{$node}->{online} = 0;
- }
- }
-
- return ($node_info, $quorate);
-};
-
-sub get_node_info {
- my ($self) = @_;
-
- my ($node_info, $quorate);
-
- my $code = sub {
- my $cstatus = $self->read_hardware_status_nolock();
- ($node_info, $quorate) = &$compute_node_info($self, $cstatus);
- };
-
- $self->global_lock($code);
-
- return ($node_info, $quorate);
-}
-
-# simulate hardware commands
-# power <node> <on|off>
-# network <node> <on|off>
-
-sub sim_hardware_cmd {
- my ($self, $cmdstr, $logid) = @_;
-
- die "implement in subclass";
-}
-
-sub run {
- my ($self) = @_;
-
- die "implement in subclass";
-}
-
-my $modify_watchog = sub {
- my ($self, $code) = @_;
-
- my $update_cmd = sub {
-
- my $filename = "$self->{statusdir}/watchdog_status";
-
- my ($res, $wdstatus);
-
- if (-f $filename) {
- my $raw = PVE::Tools::file_get_contents($filename);
- $wdstatus = decode_json($raw);
- } else {
- $wdstatus = {};
- }
-
- ($wdstatus, $res) = &$code($wdstatus);
-
- PVE::Tools::file_set_contents($filename, encode_json($wdstatus));
-
- return $res;
- };
-
- return $self->global_lock($update_cmd);
-};
-
-sub watchdog_check {
- my ($self, $node) = @_;
-
- my $code = sub {
- my ($wdstatus) = @_;
-
- my $res = 1;
-
- foreach my $wfh (keys %$wdstatus) {
- my $wd = $wdstatus->{$wfh};
- next if $wd->{node} ne $node;
-
- my $ctime = $self->get_time();
- my $tdiff = $ctime - $wd->{update_time};
-
- if ($tdiff > $watchdog_timeout) { # expired
- $res = 0;
- delete $wdstatus->{$wfh};
- }
- }
-
- return ($wdstatus, $res);
- };
-
- return &$modify_watchog($self, $code);
-}
-
-my $wdcounter = 0;
-
-sub watchdog_open {
- my ($self, $node) = @_;
-
- my $code = sub {
- my ($wdstatus) = @_;
-
- ++$wdcounter;
-
- my $id = "WD:$node:$$:$wdcounter";
-
- die "internal error" if defined($wdstatus->{$id});
-
- $wdstatus->{$id} = {
- node => $node,
- update_time => $self->get_time(),
- };
-
- return ($wdstatus, $id);
- };
-
- return &$modify_watchog($self, $code);
-}
-
-sub watchdog_close {
- my ($self, $wfh) = @_;
-
- my $code = sub {
- my ($wdstatus) = @_;
-
- my $wd = $wdstatus->{$wfh};
- die "no such watchdog handle '$wfh'\n" if !defined($wd);
-
- my $tdiff = $self->get_time() - $wd->{update_time};
- die "watchdog expired" if $tdiff > $watchdog_timeout;
-
- delete $wdstatus->{$wfh};
-
- return ($wdstatus);
- };
-
- return &$modify_watchog($self, $code);
-}
-
-sub watchdog_update {
- my ($self, $wfh) = @_;
-
- my $code = sub {
- my ($wdstatus) = @_;
-
- my $wd = $wdstatus->{$wfh};
-
- die "no such watchdog handle '$wfh'\n" if !defined($wd);
-
- my $ctime = $self->get_time();
- my $tdiff = $ctime - $wd->{update_time};
-
- die "watchdog expired" if $tdiff > $watchdog_timeout;
-
- $wd->{update_time} = $ctime;
-
- return ($wdstatus);
- };
-
- return &$modify_watchog($self, $code);
-}
-
-
-
-1;
+++ /dev/null
-SOURCES=Env.pm Hardware.pm TestEnv.pm TestHardware.pm RTEnv.pm RTHardware.pm
-
-.PHONY: installsim
-installsim:
- install -d -m 0755 ${DESTDIR}${PERLDIR}/PVE/HA/Sim
- for i in ${SOURCES}; do install -D -m 0644 $$i ${DESTDIR}${PERLDIR}/PVE/HA/Sim/$$i; done
+++ /dev/null
-package PVE::HA::Sim::RTEnv;
-
-use strict;
-use warnings;
-use POSIX qw(strftime EINTR);
-use Data::Dumper;
-use JSON;
-use IO::File;
-use Fcntl qw(:DEFAULT :flock);
-
-use PVE::HA::Tools;
-
-use base qw(PVE::HA::Sim::Env);
-
-sub new {
- my ($this, $nodename, $hardware, $log_id) = @_;
-
- my $class = ref($this) || $this;
-
- my $self = $class->SUPER::new($nodename, $hardware, $log_id);
-
- return $self;
-}
-
-sub get_time {
- my ($self) = @_;
-
- return time();
-}
-
-sub log {
- my ($self, $level, $msg) = @_;
-
- chomp $msg;
-
- my $time = $self->get_time();
-
- printf("%-5s %10s %12s: $msg\n", $level, strftime("%H:%M:%S", localtime($time)),
- "$self->{nodename}/$self->{log_id}");
-}
-
-sub sleep {
- my ($self, $delay) = @_;
-
- CORE::sleep($delay);
-}
-
-sub sleep_until {
- my ($self, $end_time) = @_;
-
- for (;;) {
- my $cur_time = time();
-
- last if $cur_time >= $end_time;
-
- $self->sleep(1);
- }
-}
-
-sub loop_start_hook {
- my ($self) = @_;
-
- $self->{loop_start} = $self->get_time();
-}
-
-sub loop_end_hook {
- my ($self) = @_;
-
- my $delay = $self->get_time() - $self->{loop_start};
-
- die "loop take too long ($delay seconds)\n" if $delay > 30;
-}
-
-sub exec_resource_agent {
- my ($self, $sid, $cmd, @params) = @_;
-
- my $hardware = $self->{hardware};
-
- my $nodename = $self->{nodename};
-
- my $sc = $hardware->read_service_config($nodename);
-
- # fixme: return valid_exit code (instead of using die)
- my $cd = $sc->{$sid};
- die "no such service" if !$cd;
-
- my $ss = $hardware->read_service_status($nodename);
-
- if ($cmd eq 'started') {
-
- # fixme: return valid_exit code
- die "service '$sid' not on this node" if $cd->{node} ne $nodename;
-
- if ($ss->{$sid}) {
- $self->log("info", "service status $sid: running");
- return 0;
- }
- $self->log("info", "starting service $sid");
-
- $self->sleep(2);
-
- $ss->{$sid} = 1;
- $hardware->write_service_status($nodename, $ss);
-
- $self->log("info", "service status $sid started");
-
- return 0;
-
- } elsif ($cmd eq 'request_stop' || $cmd eq 'stopped') {
-
- # fixme: return valid_exit code
- die "service '$sid' not on this node" if $cd->{node} ne $nodename;
-
- if (!$ss->{$sid}) {
- $self->log("info", "service status $sid: stopped");
- return 0;
- }
- $self->log("info", "stopping service $sid");
-
- $self->sleep(2);
-
- $ss->{$sid} = 0;
- $hardware->write_service_status($nodename, $ss);
-
- $self->log("info", "service status $sid stopped");
-
- return 0;
-
- } elsif ($cmd eq 'migrate' || $cmd eq 'relocate') {
-
- my $target = $params[0];
- die "$cmd '$sid' failed - missing target\n" if !defined($target);
-
- if ($cd->{node} eq $target) {
- # already migrate
- return 0;
- } elsif ($cd->{node} eq $nodename) {
-
- $self->log("info", "service $sid - start $cmd to node '$target'");
-
- if ($cmd eq 'relocate' && $ss->{$sid}) {
- $self->log("info", "stopping service $sid (relocate)");
- $self->sleep(1);
- $ss->{$sid} = 0;
- $hardware->write_service_status($nodename, $ss);
- $self->log("info", "service status $sid stopped");
- }
-
- $self->sleep(2);
- $self->change_service_location($sid, $target);
- $self->log("info", "service $sid - end $cmd to node '$target'");
-
- return 0;
-
- } else {
- die "migrate '$sid' failed - service is not on this node\n";
- }
-
-
- }
-
- die "implement me (cmd '$cmd')";
-}
-
-1;
+++ /dev/null
-package PVE::HA::Sim::RTHardware;
-
-# Simulate Hardware resources in Realtime by
-# running CRM and LRM in separate processes
-
-use strict;
-use warnings;
-use POSIX qw(strftime EINTR);
-use Data::Dumper;
-use JSON;
-use IO::File;
-use IO::Select;
-use Fcntl qw(:DEFAULT :flock);
-use File::Copy;
-use File::Path qw(make_path remove_tree);
-
-use Glib;
-
-use Gtk3 '-init';
-
-use PVE::HA::CRM;
-use PVE::HA::LRM;
-
-use PVE::HA::Sim::RTEnv;
-use base qw(PVE::HA::Sim::Hardware);
-
-sub new {
- my ($this, $testdir) = @_;
-
- my $class = ref($this) || $this;
-
- my $self = $class->SUPER::new($testdir);
-
- foreach my $node (sort keys %{$self->{nodes}}) {
- my $d = $self->{nodes}->{$node};
-
- $d->{crm} = undef; # create on power on
- $d->{lrm} = undef; # create on power on
- }
-
- $self->create_main_window();
-
- return $self;
-}
-
-sub get_time {
- my ($self) = @_;
-
- return time();
-}
-
-sub log {
- my ($self, $level, $msg, $id) = @_;
-
- chomp $msg;
-
- my $time = $self->get_time();
-
- $id = 'hardware' if !$id;
-
- my $text = sprintf("%-5s %10s %12s: $msg\n", $level,
- strftime("%H:%M:%S", localtime($time)), $id);
-
- $self->append_text($text);
-}
-
-# fixme: duplicate code in Env?
-sub read_manager_status {
- my ($self) = @_;
-
- my $filename = "$self->{statusdir}/manager_status";
-
- return PVE::HA::Tools::read_json_from_file($filename, {});
-}
-
-sub fork_daemon {
- my ($self, $lockfh, $type, $node) = @_;
-
- my @psync = POSIX::pipe();
-
- my $pid = fork();
- die "fork failed" if ! defined($pid);
-
- if ($pid == 0) {
-
- close($lockfh) if defined($lockfh); # unlock global lock
-
- POSIX::close($psync[0]);
-
- my $outfh = $psync[1];
-
- my $fd = fileno (STDIN);
- close STDIN;
- POSIX::close(0) if $fd != 0;
-
- die "unable to redirect STDIN - $!"
- if !open(STDIN, "</dev/null");
-
- # redirect STDOUT
- $fd = fileno(STDOUT);
- close STDOUT;
- POSIX::close (1) if $fd != 1;
-
- die "unable to redirect STDOUT - $!"
- if !open(STDOUT, ">&", $outfh);
-
- STDOUT->autoflush (1);
-
- # redirect STDERR to STDOUT
- $fd = fileno(STDERR);
- close STDERR;
- POSIX::close(2) if $fd != 2;
-
- die "unable to redirect STDERR - $!"
- if !open(STDERR, ">&1");
-
- STDERR->autoflush(1);
-
- if ($type eq 'crm') {
-
- my $haenv = PVE::HA::Env->new('PVE::HA::Sim::RTEnv', $node, $self, 'crm');
-
- my $crm = PVE::HA::CRM->new($haenv);
-
- for (;;) {
- $haenv->loop_start_hook();
-
- if (!$crm->do_one_iteration()) {
- $haenv->log("info", "daemon stopped");
- exit (0);
- }
-
- $haenv->loop_end_hook();
- }
-
- } else {
-
- my $haenv = PVE::HA::Env->new('PVE::HA::Sim::RTEnv', $node, $self, 'lrm');
-
- my $lrm = PVE::HA::LRM->new($haenv);
-
- for (;;) {
- $haenv->loop_start_hook();
-
- if (!$lrm->do_one_iteration()) {
- $haenv->log("info", "daemon stopped");
- exit (0);
- }
-
- $haenv->loop_end_hook();
- }
- }
-
- exit(-1);
- }
-
- # parent
-
- POSIX::close ($psync[1]);
-
- Glib::IO->add_watch($psync[0], ['in', 'hup'], sub {
- my ($fd, $cond) = @_;
- if ($cond eq 'in') {
- my $readbuf;
- if (my $count = POSIX::read($fd, $readbuf, 8192)) {
- $self->append_text($readbuf);
- }
- return 1;
- } else {
- POSIX::close($fd);
- return 0;
- }
- });
-
- return $pid;
-}
-
-# simulate hardware commands
-# power <node> <on|off>
-# network <node> <on|off>
-
-sub sim_hardware_cmd {
- my ($self, $cmdstr, $logid) = @_;
-
- my $cstatus;
-
- # note: do not fork when we own the lock!
- my $code = sub {
- my ($lockfh) = @_;
-
- $cstatus = $self->read_hardware_status_nolock();
-
- my ($cmd, $node, $action) = split(/\s+/, $cmdstr);
-
- die "sim_hardware_cmd: no node specified" if !$node;
- die "sim_hardware_cmd: unknown action '$action'" if $action !~ m/^(on|off)$/;
-
- my $d = $self->{nodes}->{$node};
- die "sim_hardware_cmd: no such node '$node'\n" if !$d;
-
- $self->log('info', "execute $cmdstr", $logid);
-
- if ($cmd eq 'power') {
- if ($cstatus->{$node}->{power} ne $action) {
- if ($action eq 'on') {
- $d->{crm} = $self->fork_daemon($lockfh, 'crm', $node) if !$d->{crm};
- $d->{lrm} = $self->fork_daemon($lockfh, 'lrm', $node) if !$d->{lrm};
- } else {
- if ($d->{crm}) {
- $self->log('info', "crm on node '$node' killed by poweroff");
- kill(9, $d->{crm});
- $d->{crm} = undef;
- }
- if ($d->{lrm}) {
- $self->log('info', "lrm on node '$node' killed by poweroff");
- kill(9, $d->{lrm});
- $d->{lrm} = undef;
- }
- }
- }
-
- $cstatus->{$node}->{power} = $action;
- $cstatus->{$node}->{network} = $action;
-
- } elsif ($cmd eq 'network') {
- $cstatus->{$node}->{network} = $action;
- } else {
- die "sim_hardware_cmd: unknown command '$cmd'\n";
- }
-
- $self->write_hardware_status_nolock($cstatus);
- };
-
- my $res = $self->global_lock($code);
-
- # update GUI outside lock
-
- foreach my $node (keys %$cstatus) {
- my $d = $self->{nodes}->{$node};
- $d->{network_btn}->set_active($cstatus->{$node}->{network} eq 'on');
- $d->{power_btn}->set_active($cstatus->{$node}->{power} eq 'on');
- }
-
- return $res;
-}
-
-sub cleanup {
- my ($self) = @_;
-
- my @nodes = sort keys %{$self->{nodes}};
- foreach my $node (@nodes) {
- my $d = $self->{nodes}->{$node};
-
- if ($d->{crm}) {
- kill 9, $d->{crm};
- delete $d->{crm};
- }
- if ($d->{lrm}) {
- kill 9, $d->{lrm};
- delete $d->{lrm};
- }
- }
-}
-
-sub append_text {
- my ($self, $text) = @_;
-
- my $logview = $self->{gui}->{text_view} || die "GUI not ready";
- my $textbuf = $logview->get_buffer();
-
- $textbuf->insert_at_cursor($text, -1);
- my $lines = $textbuf->get_line_count();
-
- my $history = 102;
-
- if ($lines > $history) {
- my $start = $textbuf->get_iter_at_line(0);
- my $end = $textbuf->get_iter_at_line($lines - $history);
- $textbuf->delete($start, $end);
- }
-
- $logview->scroll_to_mark($textbuf->get_insert(), 0.0, 1, 0.0, 1.0);
-}
-
-sub set_power_state {
- my ($self, $node) = @_;
-
- my $d = $self->{nodes}->{$node} || die "no such node '$node'";
-
- my $action = $d->{power_btn}->get_active() ? 'on' : 'off';
-
- $self->sim_hardware_cmd("power $node $action");
-}
-
-sub set_network_state {
- my ($self, $node) = @_;
-
- my $d = $self->{nodes}->{$node} || die "no such node '$node'";
-
- my $action = $d->{network_btn}->get_active() ? 'on' : 'off';
-
- $self->sim_hardware_cmd("network $node $action");
-}
-
-sub set_service_state {
- my ($self, $sid) = @_;
-
- $self->{service_config} = $self->read_service_config();
-
- my $d = $self->{service_gui}->{$sid} || die "no such service '$sid'";
-
- my $state = $d->{enable_btn}->get_active() ? 'enabled' : 'disabled';
-
- $d = $self->{service_config}->{$sid} || die "no such service '$sid'";
-
- $d->{state} = $state;
-
- $self->write_service_config($self->{service_config});
-}
-
-sub create_node_control {
- my ($self) = @_;
-
- my $ngrid = Gtk3::Grid->new();
- $ngrid->set_row_spacing(2);
- $ngrid->set_column_spacing(5);
- $ngrid->set('margin-left', 5);
-
- my $w = Gtk3::Label->new('Node');
- $ngrid->attach($w, 0, 0, 1, 1);
- $w = Gtk3::Label->new('Power');
- $ngrid->attach($w, 1, 0, 1, 1);
- $w = Gtk3::Label->new('Network');
- $ngrid->attach($w, 2, 0, 1, 1);
- $w = Gtk3::Label->new('Status');
- $w->set_size_request(150, -1);
- $w->set_alignment (0, 0.5);
- $ngrid->attach($w, 3, 0, 1, 1);
-
- my $row = 1;
-
- my @nodes = sort keys %{$self->{nodes}};
-
- foreach my $node (@nodes) {
- my $d = $self->{nodes}->{$node};
-
- $w = Gtk3::Label->new($node);
- $ngrid->attach($w, 0, $row, 1, 1);
- $w = Gtk3::Switch->new();
- $ngrid->attach($w, 1, $row, 1, 1);
- $d->{power_btn} = $w;
- $w->signal_connect('notify::active' => sub {
- $self->set_power_state($node);
- }),
-
- $w = Gtk3::Switch->new();
- $ngrid->attach($w, 2, $row, 1, 1);
- $d->{network_btn} = $w;
- $w->signal_connect('notify::active' => sub {
- $self->set_network_state($node);
- }),
-
- $w = Gtk3::Label->new('-');
- $w->set_alignment (0, 0.5);
- $ngrid->attach($w, 3, $row, 1, 1);
- $d->{node_status_label} = $w;
-
- $row++;
- }
-
- return $ngrid;
-}
-
-sub show_migrate_dialog {
- my ($self, $sid) = @_;
-
- my $dialog = Gtk3::Dialog->new();
-
- $dialog->set_title("Migrate $sid");
- $dialog->set_modal(1);
-
- my $grid = Gtk3::Grid->new();
- $grid->set_row_spacing(2);
- $grid->set_column_spacing(5);
- $grid->set('margin', 5);
-
- my $w = Gtk3::Label->new('Target Mode');
- $grid->attach($w, 0, 0, 1, 1);
-
- my @nodes = sort keys %{$self->{nodes}};
- $w = Gtk3::ComboBoxText->new();
- foreach my $node (@nodes) {
- $w->append_text($node);
- }
-
- my $target = '';
- $w->signal_connect('notify::active' => sub {
- my $w = shift;
-
- my $sel = $w->get_active();
- return if $sel < 0;
-
- $target = $nodes[$sel];
- });
- $grid->attach($w, 1, 0, 1, 1);
-
- my $relocate_btn = Gtk3::CheckButton->new_with_label("stop service (relocate)");
- $grid->attach($relocate_btn, 1, 1, 1, 1);
-
- my $contarea = $dialog->get_content_area();
-
- $contarea->add($grid);
-
- $dialog->add_button("_OK", 1);
-
- $dialog->show_all();
- my $res = $dialog->run();
-
- $dialog->destroy();
-
- if ($res == 1 && $target) {
- if ($relocate_btn->get_active()) {
- $self->queue_crm_commands("relocate $sid $target");
- } else {
- $self->queue_crm_commands("migrate $sid $target");
- }
- }
-}
-
-sub create_service_control {
- my ($self) = @_;
-
- my $sgrid = Gtk3::Grid->new();
- $sgrid->set_row_spacing(2);
- $sgrid->set_column_spacing(5);
- $sgrid->set('margin', 5);
-
- my $w = Gtk3::Label->new('Service');
- $sgrid->attach($w, 0, 0, 1, 1);
- $w = Gtk3::Label->new('Enable');
- $sgrid->attach($w, 1, 0, 1, 1);
- $w = Gtk3::Label->new('Node');
- $sgrid->attach($w, 3, 0, 1, 1);
- $w = Gtk3::Label->new('Status');
- $w->set_alignment (0, 0.5);
- $w->set_size_request(150, -1);
- $sgrid->attach($w, 4, 0, 1, 1);
-
- my $row = 1;
- my @nodes = keys %{$self->{nodes}};
-
- foreach my $sid (sort keys %{$self->{service_config}}) {
- my $d = $self->{service_config}->{$sid};
-
- $w = Gtk3::Label->new($sid);
- $sgrid->attach($w, 0, $row, 1, 1);
-
- $w = Gtk3::Switch->new();
- $sgrid->attach($w, 1, $row, 1, 1);
- $w->set_active(1) if $d->{state} eq 'enabled';
- $self->{service_gui}->{$sid}->{enable_btn} = $w;
- $w->signal_connect('notify::active' => sub {
- $self->set_service_state($sid);
- }),
-
-
- $w = Gtk3::Button->new('Migrate');
- $sgrid->attach($w, 2, $row, 1, 1);
- $w->signal_connect(clicked => sub {
- $self->show_migrate_dialog($sid);
- });
-
- $w = Gtk3::Label->new($d->{node});
- $sgrid->attach($w, 3, $row, 1, 1);
- $self->{service_gui}->{$sid}->{node_label} = $w;
-
- $w = Gtk3::Label->new('-');
- $w->set_alignment (0, 0.5);
- $sgrid->attach($w, 4, $row, 1, 1);
- $self->{service_gui}->{$sid}->{status_label} = $w;
-
- $row++;
- }
-
- return $sgrid;
-}
-
-sub create_log_view {
- my ($self) = @_;
-
- my $nb = Gtk3::Notebook->new();
-
- my $l1 = Gtk3::Label->new('Cluster Log');
-
- my $logview = Gtk3::TextView->new();
- $logview->set_editable(0);
- $logview->set_cursor_visible(0);
-
- $self->{gui}->{text_view} = $logview;
-
- my $swindow = Gtk3::ScrolledWindow->new();
- $swindow->set_size_request(1024, 768);
- $swindow->add($logview);
-
- $nb->insert_page($swindow, $l1, 0);
-
- my $l2 = Gtk3::Label->new('Manager Status');
-
- my $statview = Gtk3::TextView->new();
- $statview->set_editable(0);
- $statview->set_cursor_visible(0);
-
- $self->{gui}->{stat_view} = $statview;
-
- $swindow = Gtk3::ScrolledWindow->new();
- $swindow->set_size_request(640, 400);
- $swindow->add($statview);
-
- $nb->insert_page($swindow, $l2, 1);
- return $nb;
-}
-
-sub create_main_window {
- my ($self) = @_;
-
- my $window = Gtk3::Window->new();
- $window->set_title("Proxmox HA Simulator");
-
- $window->signal_connect( destroy => sub { Gtk3::main_quit(); });
-
- my $grid = Gtk3::Grid->new();
-
- my $frame = $self->create_log_view();
- $grid->attach($frame, 0, 0, 1, 1);
- $frame->set('expand', 1);
-
- my $vbox = Gtk3::VBox->new(0, 0);
- $grid->attach($vbox, 1, 0, 1, 1);
-
- my $ngrid = $self->create_node_control();
- $vbox->pack_start($ngrid, 0, 0, 0);
-
- my $sep = Gtk3::HSeparator->new;
- $sep->set('margin-top', 10);
- $vbox->pack_start ($sep, 0, 0, 0);
-
- my $sgrid = $self->create_service_control();
- $vbox->pack_start($sgrid, 0, 0, 0);
-
- $window->add($grid);
-
- $window->show_all;
- $window->realize ();
-}
-
-sub run {
- my ($self) = @_;
-
- Glib::Timeout->add(1000, sub {
-
- $self->{service_config} = $self->read_service_config();
-
- # check all watchdogs
- my @nodes = sort keys %{$self->{nodes}};
- foreach my $node (@nodes) {
- if (!$self->watchdog_check($node)) {
- $self->sim_hardware_cmd("power $node off", 'watchdog');
- $self->log('info', "server '$node' stopped by poweroff (watchdog)");
- }
- }
-
- my $mstatus = $self->read_manager_status();
- my $node_status = $mstatus->{node_status} || {};
-
- foreach my $node (@nodes) {
- my $ns = $node_status->{$node} || '-';
- my $d = $self->{nodes}->{$node};
- next if !$d;
- my $sl = $d->{node_status_label};
- next if !$sl;
-
- if ($mstatus->{master_node} && ($mstatus->{master_node} eq $node)) {
- $sl->set_text(uc($ns));
- } else {
- $sl->set_text($ns);
- }
- }
-
- my $service_status = $mstatus->{service_status} || {};
- my @services = sort keys %{$self->{service_config}};
-
- foreach my $sid (@services) {
- my $sc = $self->{service_config}->{$sid};
- my $ss = $service_status->{$sid};
- my $sgui = $self->{service_gui}->{$sid};
- next if !$sgui;
- my $nl = $sgui->{node_label};
- $nl->set_text($sc->{node});
-
- my $sl = $sgui->{status_label};
- next if !$sl;
-
- my $text = ($ss && $ss->{state}) ? $ss->{state} : '-';
- $sl->set_text($text);
- }
-
- if (my $sv = $self->{gui}->{stat_view}) {
- my $text = Dumper($mstatus);
- my $textbuf = $sv->get_buffer();
- $textbuf->set_text($text, -1);
- }
-
- return 1; # repeat
- });
-
- Gtk3->main;
-
- $self->cleanup();
-}
-
-1;
+++ /dev/null
-package PVE::HA::Sim::TestEnv;
-
-use strict;
-use warnings;
-use POSIX qw(strftime EINTR);
-use Data::Dumper;
-use JSON;
-use IO::File;
-use Fcntl qw(:DEFAULT :flock);
-
-use PVE::HA::Tools;
-
-use base qw(PVE::HA::Sim::Env);
-
-sub new {
- my ($this, $nodename, $hardware, $log_id) = @_;
-
- my $class = ref($this) || $this;
-
- my $self = $class->SUPER::new($nodename, $hardware, $log_id);
-
- $self->{cur_time} = 0;
- $self->{loop_delay} = 0;
-
- return $self;
-}
-
-sub get_time {
- my ($self) = @_;
-
- return $self->{cur_time};
-}
-
-sub sleep {
- my ($self, $delay) = @_;
-
- $self->{loop_delay} += $delay;
-}
-
-sub sleep_until {
- my ($self, $end_time) = @_;
-
- my $cur_time = $self->{cur_time} + $self->{loop_delay};
-
- return if $cur_time >= $end_time;
-
- $self->{loop_delay} += $end_time - $cur_time;
-}
-
-sub get_ha_manager_lock {
- my ($self) = @_;
-
- my $res = $self->SUPER::get_ha_manager_lock();
- ++$self->{loop_delay};
- return $res;
-}
-
-sub get_ha_agent_lock {
- my ($self) = @_;
-
- my $res = $self->SUPER::get_ha_agent_lock();
- ++$self->{loop_delay};
-
- return $res;
-}
-
-sub test_ha_agent_lock {
- my ($self, $node) = @_;
-
- my $res = $self->SUPER::test_ha_agent_lock($node);
- ++$self->{loop_delay};
- return $res;
-}
-
-sub loop_start_hook {
- my ($self, $starttime) = @_;
-
- $self->{loop_delay} = 0;
-
- die "no starttime" if !defined($starttime);
- die "strange start time" if $starttime < $self->{cur_time};
-
- $self->{cur_time} = $starttime;
-
- # do nothing
-}
-
-sub loop_end_hook {
- my ($self) = @_;
-
- my $delay = $self->{loop_delay};
- $self->{loop_delay} = 0;
-
- die "loop take too long ($delay seconds)\n" if $delay > 30;
-
- # $self->{cur_time} += $delay;
-
- $self->{cur_time} += 1; # easier for simulation
-}
-
-1;
+++ /dev/null
-package PVE::HA::Sim::TestHardware;
-
-# Simulate Hardware resources
-
-# power supply for nodes: on/off
-# network connection to nodes: on/off
-# watchdog devices for nodes
-
-use strict;
-use warnings;
-use POSIX qw(strftime EINTR);
-use Data::Dumper;
-use JSON;
-use IO::File;
-use Fcntl qw(:DEFAULT :flock);
-use File::Copy;
-use File::Path qw(make_path remove_tree);
-
-use PVE::HA::CRM;
-use PVE::HA::LRM;
-
-use PVE::HA::Sim::TestEnv;
-use base qw(PVE::HA::Sim::Hardware);
-
-my $max_sim_time = 10000;
-
-sub new {
- my ($this, $testdir) = @_;
-
- my $class = ref($this) || $this;
-
- my $self = $class->SUPER::new($testdir);
-
- my $raw = PVE::Tools::file_get_contents("$testdir/cmdlist");
- $self->{cmdlist} = decode_json($raw);
-
- $self->{loop_count} = 0;
- $self->{cur_time} = 0;
-
- foreach my $node (sort keys %{$self->{nodes}}) {
-
- my $d = $self->{nodes}->{$node};
-
- $d->{crm_env} =
- PVE::HA::Env->new('PVE::HA::Sim::TestEnv', $node, $self, 'crm');
-
- $d->{lrm_env} =
- PVE::HA::Env->new('PVE::HA::Sim::TestEnv', $node, $self, 'lrm');
-
- $d->{crm} = undef; # create on power on
- $d->{lrm} = undef; # create on power on
- }
-
- return $self;
-}
-
-sub get_time {
- my ($self) = @_;
-
- return $self->{cur_time};
-}
-
-# simulate hardware commands
-# power <node> <on|off>
-# network <node> <on|off>
-
-sub sim_hardware_cmd {
- my ($self, $cmdstr, $logid) = @_;
-
- my $code = sub {
-
- my $cstatus = $self->read_hardware_status_nolock();
-
- my ($cmd, $node, $action) = split(/\s+/, $cmdstr);
-
- die "sim_hardware_cmd: no node specified" if !$node;
- die "sim_hardware_cmd: unknown action '$action'" if $action !~ m/^(on|off)$/;
-
- my $d = $self->{nodes}->{$node};
- die "sim_hardware_cmd: no such node '$node'\n" if !$d;
-
- $self->log('info', "execute $cmdstr", $logid);
-
- if ($cmd eq 'power') {
- if ($cstatus->{$node}->{power} ne $action) {
- if ($action eq 'on') {
- $d->{crm} = PVE::HA::CRM->new($d->{crm_env}) if !$d->{crm};
- $d->{lrm} = PVE::HA::LRM->new($d->{lrm_env}) if !$d->{lrm};
- } else {
- if ($d->{crm}) {
- $d->{crm_env}->log('info', "killed by poweroff");
- $d->{crm} = undef;
- }
- if ($d->{lrm}) {
- $d->{lrm_env}->log('info', "killed by poweroff");
- $d->{lrm} = undef;
- }
- }
- }
-
- $cstatus->{$node}->{power} = $action;
- $cstatus->{$node}->{network} = $action;
-
- } elsif ($cmd eq 'network') {
- $cstatus->{$node}->{network} = $action;
- } else {
- die "sim_hardware_cmd: unknown command '$cmd'\n";
- }
-
- $self->write_hardware_status_nolock($cstatus);
- };
-
- return $self->global_lock($code);
-}
-
-sub run {
- my ($self) = @_;
-
- my $last_command_time = 0;
-
- for (;;) {
-
- my $starttime = $self->get_time();
-
- my @nodes = sort keys %{$self->{nodes}};
-
- my $nodecount = scalar(@nodes);
-
- my $looptime = $nodecount*2;
- $looptime = 20 if $looptime < 20;
-
- die "unable to simulate so many nodes. You need to increate watchdog/lock timeouts.\n"
- if $looptime >= 60;
-
- foreach my $node (@nodes) {
-
- my $d = $self->{nodes}->{$node};
-
- if (my $crm = $d->{crm}) {
-
- $d->{crm_env}->loop_start_hook($self->get_time());
-
- die "implement me (CRM exit)" if !$crm->do_one_iteration();
-
- $d->{crm_env}->loop_end_hook();
-
- my $nodetime = $d->{crm_env}->get_time();
- $self->{cur_time} = $nodetime if $nodetime > $self->{cur_time};
- }
-
- if (my $lrm = $d->{lrm}) {
-
- $d->{lrm_env}->loop_start_hook($self->get_time());
-
- die "implement me (LRM exit)" if !$lrm->do_one_iteration();
-
- $d->{lrm_env}->loop_end_hook();
-
- my $nodetime = $d->{lrm_env}->get_time();
- $self->{cur_time} = $nodetime if $nodetime > $self->{cur_time};
- }
-
- foreach my $n (@nodes) {
- if (!$self->watchdog_check($n)) {
- $self->sim_hardware_cmd("power $n off", 'watchdog');
- $self->log('info', "server '$n' stopped by poweroff (watchdog)");
- $self->{nodes}->{$n}->{crm} = undef;
- $self->{nodes}->{$n}->{lrm} = undef;
- }
- }
- }
-
-
- $self->{cur_time} = $starttime + $looptime
- if ($self->{cur_time} - $starttime) < $looptime;
-
- die "simulation end\n" if $self->{cur_time} > $max_sim_time;
-
- # apply new comand after 5 loop iterations
-
- if (($self->{loop_count} % 5) == 0) {
- my $list = shift $self->{cmdlist};
- if (!$list) {
- # end sumulation (500 seconds after last command)
- return if (($self->{cur_time} - $last_command_time) > 500);
- }
-
- foreach my $cmd (@$list) {
- $last_command_time = $self->{cur_time};
- $self->sim_hardware_cmd($cmd, 'cmdlist');
- }
- }
-
- ++$self->{loop_count};
- }
-}
-
-1;
+++ /dev/null
-package PVE::HA::Tools;
-
-use strict;
-use warnings;
-use JSON;
-use PVE::Tools;
-
-sub read_json_from_file {
- my ($filename, $default) = @_;
-
- my $data;
-
- if (defined($default) && (! -f $filename)) {
- $data = $default;
- } else {
- my $raw = PVE::Tools::file_get_contents($filename);
- $data = decode_json($raw);
- }
-
- return $data;
-}
-
-sub write_json_to_file {
- my ($filename, $data) = @_;
-
- my $raw = encode_json($data);
-
- PVE::Tools::file_set_contents($filename, $raw);
-}
-
-
-1;
+++ /dev/null
-
-.PHONY: install
-install:
- install -d -m 0755 ${DESTDIR}${PERLDIR}/PVE
- make -C HA install
-
-.PHONY: installsim
-installsim:
- install -d -m 0755 ${DESTDIR}${PERLDIR}/PVE
- make -C HA installsim
+++ /dev/null
-Package: pve-ha-manager
-Version: @@VERSION@@-@@PKGRELEASE@@
-Section: perl
-Priority: optional
-Architecture: @@ARCH@@
-Depends: perl (>= 5.14.2-21), libpve-common-perl, pve-cluster (>= 3.0-17)
-Maintainer: Proxmox Support Team <support@proxmox.com>
-Description: Proxmox VE HA Manager
- HA Manager Proxmox VE.
+++ /dev/null
-Copyright (C) 2015 Proxmox Server Solutions GmbH
-
-This software is written by Proxmox Server Solutions GmbH <support@proxmox.com>
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program. If not, see <http://www.gnu.org/licenses/>.
--- /dev/null
+Source: pve-ha-manager
+Section: perl
+Priority: extra
+Maintainer: Proxmox Support Team <support@proxmox.com>
+Build-Depends: debhelper (>= 7.0.50~), libpve-common-perl, libglib-perl, libgtk3-perl
+Standards-Version: 3.8.4
+
+Package: pve-ha-manager
+Section: perl
+Priority: optional
+Architecture: any
+Depends: ${perl:Depends}, ${misc:Depends}, libpve-common-perl, pve-cluster (>= 3.0-17)
+Description: Proxmox VE HA Manager
+ HA Manager Proxmox VE.
--- /dev/null
+Copyright (C) 2015 Proxmox Server Solutions GmbH
+
+This software is written by Proxmox Server Solutions GmbH <support@proxmox.com>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
--- /dev/null
+#!/usr/bin/make -f
+# -*- makefile -*-
+# Sample debian/rules that uses debhelper.
+# This file was originally written by Joey Hess and Craig Small.
+# As a special exception, when this file is copied by dh-make into a
+# dh-make output file, you may use that output file without restriction.
+# This special exception was added by Craig Small in version 0.37 of dh-make.
+
+# Uncomment this to turn on verbose mode.
+#export DH_VERBOSE=1
+
+%:
+ dh $@
--- /dev/null
+[Service]
+ExecStart=/usr/sbin/watchdog-mux
--- /dev/null
+[Socket]
+ListenStream=/run/watchdog-mux.sock
+
+[Install]
+WantedBy=sockets.target
+++ /dev/null
-#!/usr/bin/perl
-
-use strict;
-use warnings;
-use PVE::SafeSyslog;
-use PVE::Daemon;
-use Data::Dumper;
-use PVE::RPCEnvironment;
-
-use PVE::HA::Env;
-use PVE::HA::Env::PVE2;
-use PVE::HA::CRM;
-
-use base qw(PVE::Daemon);
-
-my $cmdline = [$0, @ARGV];
-
-my %daemon_options = (stop_wait_time => 5);
-
-my $daemon = __PACKAGE__->new('pve-ha-crm', $cmdline, %daemon_options);
-
-my $rpcenv = PVE::RPCEnvironment->init('cli');
-
-$rpcenv->init_request();
-$rpcenv->set_language($ENV{LANG});
-$rpcenv->set_user('root@pam');
-
-sub run {
- my ($self) = @_;
-
- $self->{haenv} = PVE::HA::Env->new('PVE::HA::Env::PVE2', $self->{nodename});
-
- $self->{crm} = PVE::HA::CRM->new($self->{haenv});
-
- for (;;) {
- $self->{haenv}->loop_start_hook();
-
- my $repeat = $self->{crm}->do_one_iteration();
-
- $self->{haenv}->loop_end_hook();
-
- last if !$repeat;
- }
-}
-
-sub shutdown {
- my ($self) = @_;
-
- $self->{crm}->shutdown_request();
-}
-
-$daemon->register_start_command();
-$daemon->register_stop_command();
-$daemon->register_status_command();
-
-my $cmddef = {
- start => [ __PACKAGE__, 'start', []],
- stop => [ __PACKAGE__, 'stop', []],
- status => [ __PACKAGE__, 'status', [], undef, sub { print shift . "\n";} ],
-};
-
-my $cmd = shift;
-
-PVE::CLIHandler::handle_cmd($cmddef, $0, $cmd, \@ARGV, undef, $0);
-
-exit (0);
-
-__END__
-
-=head1 NAME
-
-pve-ha-crm - PVE Cluster Ressource Manager Daemon
-
-=head1 SYNOPSIS
-
-=include synopsis
-
-=head1 DESCRIPTION
-
-This is the Cluster Ressource Manager.
-
-=include pve_copyright
+++ /dev/null
-#!/usr/bin/perl
-
-use strict;
-use warnings;
-use PVE::SafeSyslog;
-use PVE::Daemon;
-use Data::Dumper;
-use PVE::RPCEnvironment;
-
-use PVE::HA::Env;
-use PVE::HA::Env::PVE2;
-use PVE::HA::LRM;
-
-use base qw(PVE::Daemon);
-
-my $cmdline = [$0, @ARGV];
-
-my %daemon_options = (stop_wait_time => 60);
-
-my $daemon = __PACKAGE__->new('pve-ha-lrm', $cmdline, %daemon_options);
-
-my $rpcenv = PVE::RPCEnvironment->init('cli');
-
-$rpcenv->init_request();
-$rpcenv->set_language($ENV{LANG});
-$rpcenv->set_user('root@pam');
-
-sub run {
- my ($self) = @_;
-
- $self->{haenv} = PVE::HA::Env->new('PVE::HA::Env::PVE2', $self->{nodename});
-
- $self->{lrm} = PVE::HA::LRM->new($self->{haenv});
-
- for (;;) {
- $self->{haenv}->loop_start_hook();
-
- my $repeat = $self->{lrm}->do_one_iteration();
-
- $self->{haenv}->loop_end_hook();
-
- last if !$repeat;
- }
-}
-
-sub shutdown {
- my ($self) = @_;
-
- $self->{lrm}->shutdown_request();
-}
-
-$daemon->register_start_command();
-$daemon->register_stop_command();
-$daemon->register_status_command();
-
-my $cmddef = {
- start => [ __PACKAGE__, 'start', []],
- stop => [ __PACKAGE__, 'stop', []],
- status => [ __PACKAGE__, 'status', [], undef, sub { print shift . "\n";} ],
-};
-
-my $cmd = shift;
-
-PVE::CLIHandler::handle_cmd($cmddef, $0, $cmd, \@ARGV, undef, $0);
-
-exit (0);
-
-__END__
-
-=head1 NAME
-
-pve-ha-lrm - PVE Local HA Ressource Manager Daemon
-
-=head1 SYNOPSIS
-
-=include synopsis
-
-=head1 DESCRIPTION
-
-This is the Local HA Ressource Manager.
-
-=include pve_copyright
+++ /dev/null
-#!/usr/bin/perl
-
-use strict;
-use warnings;
-use lib '/usr/share/pve-ha-simulator';
-use Getopt::Long;
-use JSON;
-
-use PVE::Tools;
-use PVE::HA::Sim::TestHardware;
-use PVE::HA::Sim::RTHardware;
-
-my $opt_batch;
-
-sub show_usage {
- print "usage: $0 <testdir> [--batch]\n";
- exit(-1);
-};
-
-if (!GetOptions ("batch" => \$opt_batch)) {
- show_usage();
-}
-
-my $testdir = shift || show_usage();
-
-my $hardware;
-
-if ($opt_batch) {
- $hardware = PVE::HA::Sim::TestHardware->new($testdir);
-} else {
- $hardware = PVE::HA::Sim::RTHardware->new($testdir);
-}
-
-$hardware->log('info', "starting simulation");
-
-eval { $hardware->run(); };
-if (my $err = $@) {
- $hardware->log('err', "exit simulation - $err ");
-} else {
- $hardware->log('info', "exit simulation - done");
-}
-
-exit(0);
-
-
-
+++ /dev/null
-Package: pve-ha-simulator
-Version: @@VERSION@@-@@PKGRELEASE@@
-Section: perl
-Priority: optional
-Architecture: all
-Depends: perl (>= 5.14.2-21), libpve-common-perl, libglib-perl, libgtk3-perl
-Maintainer: Proxmox Support Team <support@proxmox.com>
-Description: Proxmox VE HA Simulator
- This is a simple GUI to simulate the bebavior of a Proxmox VE HA cluster.
--- /dev/null
+Source: pve-ha-manager
+Section: perl
+Priority: extra
+Maintainer: Proxmox Support Team <support@proxmox.com>
+Build-Depends: debhelper (>= 7.0.50~), libpve-common-perl, libglib-perl, libgtk3-perl
+Standards-Version: 3.8.4
+
+Package: pve-ha-simulator
+Section: perl
+Priority: optional
+Architecture: all
+Depends: ${perl:Depends}, ${misc:Depends}, libpve-common-perl, libglib-perl, libgtk3-perl
+Description: Proxmox VE HA Simulator
+ This is a simple GUI to simulate the bebavior of a Proxmox VE HA cluster.
--- /dev/null
+Copyright (C) 2015 Proxmox Server Solutions GmbH
+
+This software is written by Proxmox Server Solutions GmbH <support@proxmox.com>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
--- /dev/null
+#!/usr/bin/make -f
+# -*- makefile -*-
+# Sample debian/rules that uses debhelper.
+# This file was originally written by Joey Hess and Craig Small.
+# As a special exception, when this file is copied by dh-make into a
+# dh-make output file, you may use that output file without restriction.
+# This special exception was added by Craig Small in version 0.37 of dh-make.
+
+# Uncomment this to turn on verbose mode.
+#export DH_VERBOSE=1
+
+%:
+ dh $@
+
+override_dh_auto_install:
+
+ make DESTDIR=$$(pwd)/debian/pve-ha-simulator installsim
--- /dev/null
+PACKAGE=pve-ha-manager
+SIMPACKAGE=pve-ha-simulator
+
+PREFIX=/usr
+BINDIR=${PREFIX}/bin
+SBINDIR=${PREFIX}/sbin
+MANDIR=${PREFIX}/share/man
+DOCDIR=${PREFIX}/share/doc/${PACKAGE}
+SIMDOCDIR=${PREFIX}/share/doc/${SIMPACKAGE}
+PODDIR=${DOCDIR}/pod
+MAN1DIR=${MANDIR}/man1/
+export PERLDIR=${PREFIX}/share/perl5
+
+all: watchdog-mux
+
+%.1.gz: %.1.pod
+ rm -f $@
+ cat $<|pod2man -n $* -s 1 -r ${VERSION} -c "Proxmox Documentation"|gzip -c9 >$@
+
+pve-ha-crm.1.pod: pve-ha-crm
+ perl -I. ./pve-ha-crm printmanpod >$@
+
+pve-ha-lrm.1.pod: pve-ha-lrm
+ perl -I. ./pve-ha-lrm printmanpod >$@
+
+watchdog-mux: watchdog-mux.c
+ gcc watchdog-mux.c -o watchdog-mux -Wall $$(pkg-config --libs --cflags libsystemd-daemon)
+
+.PHONY: install
+install: pve-ha-crm pve-ha-lrm pve-ha-crm.1.pod pve-ha-crm.1.gz pve-ha-lrm.1.pod pve-ha-lrm.1.gz
+ install -d ${DESTDIR}${SBINDIR}
+ install -m 0755 pve-ha-crm ${DESTDIR}${SBINDIR}
+ install -m 0755 pve-ha-lrm ${DESTDIR}${SBINDIR}
+ make -C PVE install
+ install -d ${DESTDIR}/usr/share/man/man1
+ install -d ${DESTDIR}${PODDIR}
+ install -m 0644 pve-ha-crm.1.gz ${DESTDIR}/usr/share/man/man1/
+ install -m 0644 pve-ha-crm.1.pod ${DESTDIR}/${PODDIR}
+ install -m 0644 pve-ha-lrm.1.gz ${DESTDIR}/usr/share/man/man1/
+ install -m 0644 pve-ha-lrm.1.pod ${DESTDIR}/${PODDIR}
+
+.PHONY: installsim
+installsim: pve-ha-simulator
+ install -d ${DESTDIR}${SBINDIR}
+ install -m 0755 pve-ha-simulator ${DESTDIR}${SBINDIR}
+ make -C PVE PERLDIR=${PREFIX}/share/${SIMPACKAGE} installsim
+
+.PHONY: simdeb ${SIMDEB}
+simdeb ${SIMDEB}:
+ rm -rf build
+ mkdir build
+ make DESTDIR=${CURDIR}/build PERLDIR=${PREFIX}/share/${SIMPACKAGE} installsim
+ perl -I. ./pve-ha-crm verifyapi
+ perl -I. ./pve-ha-lrm verifyapi
+ install -d -m 0755 build/DEBIAN
+ sed -e s/@@VERSION@@/${VERSION}/ -e s/@@PKGRELEASE@@/${PKGREL}/ -e s/@@ARCH@@/${ARCH}/ <simcontrol.in >build/DEBIAN/control
+ install -D -m 0644 copyright build/${SIMDOCDIR}/copyright
+ install -m 0644 changelog.Debian build/${SIMDOCDIR}/
+ gzip -9 build/${SIMDOCDIR}/changelog.Debian
+ echo "git clone git://git.proxmox.com/git/pve-storage.git\\ngit checkout ${GITVERSION}" > build/${SIMDOCDIR}/SOURCE
+ dpkg-deb --build build
+ mv build.deb ${SIMDEB}
+ rm -rf debian
+ lintian ${SIMDEB}
+
+.PHONY: deb ${DEB}
+deb ${DEB}:
+ rm -rf build
+ mkdir build
+ make DESTDIR=${CURDIR}/build install
+ perl -I. ./pve-ha-crm verifyapi
+ perl -I. ./pve-ha-lrm verifyapi
+ install -d -m 0755 build/DEBIAN
+ sed -e s/@@VERSION@@/${VERSION}/ -e s/@@PKGRELEASE@@/${PKGREL}/ -e s/@@ARCH@@/${ARCH}/ <control.in >build/DEBIAN/control
+ install -D -m 0644 copyright build/${DOCDIR}/copyright
+ install -m 0644 changelog.Debian build/${DOCDIR}/
+ gzip -9 build/${DOCDIR}/changelog.Debian
+ echo "git clone git://git.proxmox.com/git/pve-storage.git\\ngit checkout ${GITVERSION}" > build/${DOCDIR}/SOURCE
+ dpkg-deb --build build
+ mv build.deb ${DEB}
+ rm -rf debian
+ lintian ${DEB}
+
+
+.PHONY: test
+test:
+# make -C test test
+
+.PHONY: clean
+clean:
+ make -C test clean
+ rm -rf watchdog-mux *.1.pod *.1.gz
+ find . -name '*~' -exec rm {} ';'
+
+.PHONY: distclean
+distclean: clean
+
--- /dev/null
+package PVE::HA::CRM;
+
+# Cluster Resource Manager
+
+use strict;
+use warnings;
+
+use PVE::SafeSyslog;
+use PVE::Tools;
+use PVE::HA::Tools;
+
+use PVE::HA::Manager;
+
+# Server can have several state:
+
+my $valid_states = {
+ wait_for_quorum => "cluster is not quorate, waiting",
+ master => "quorate, and we got the ha_manager lock",
+ lost_manager_lock => "we lost the ha_manager lock (watchgog active)",
+ slave => "quorate, but we do not own the ha_manager lock",
+};
+
+sub new {
+ my ($this, $haenv) = @_;
+
+ my $class = ref($this) || $this;
+
+ my $self = bless {
+ haenv => $haenv,
+ manager => undef,
+ status => { state => 'startup' },
+ }, $class;
+
+ $self->set_local_status({ state => 'wait_for_quorum' });
+
+ return $self;
+}
+
+sub shutdown_request {
+ my ($self) = @_;
+
+ syslog('info' , "server received shutdown request")
+ if !$self->{shutdown_request};
+
+ $self->{shutdown_request} = 1;
+}
+
+sub get_local_status {
+ my ($self) = @_;
+
+ return $self->{status};
+}
+
+sub set_local_status {
+ my ($self, $new) = @_;
+
+ die "invalid state '$new->{state}'" if !$valid_states->{$new->{state}};
+
+ my $haenv = $self->{haenv};
+
+ my $old = $self->{status};
+
+ # important: only update if if really changed
+ return if $old->{state} eq $new->{state};
+
+ $haenv->log('info', "status change $old->{state} => $new->{state}");
+
+ $new->{state_change_time} = $haenv->get_time();
+
+ $self->{status} = $new;
+
+ # fixme: do not use extra class
+ if ($new->{state} eq 'master') {
+ $self->{manager} = PVE::HA::Manager->new($haenv);
+ } else {
+ if ($self->{manager}) {
+ # fixme: what should we do here?
+ $self->{manager}->cleanup();
+ $self->{manager} = undef;
+ }
+ }
+}
+
+sub get_protected_ha_manager_lock {
+ my ($self) = @_;
+
+ my $haenv = $self->{haenv};
+
+ my $count = 0;
+ my $starttime = $haenv->get_time();
+
+ for (;;) {
+
+ if ($haenv->get_ha_manager_lock()) {
+ if ($self->{ha_manager_wd}) {
+ $haenv->watchdog_update($self->{ha_manager_wd});
+ } else {
+ my $wfh = $haenv->watchdog_open();
+ $self->{ha_manager_wd} = $wfh;
+ }
+ return 1;
+ }
+
+ last if ++$count > 5; # try max 5 time
+
+ my $delay = $haenv->get_time() - $starttime;
+ last if $delay > 5; # for max 5 seconds
+
+ $haenv->sleep(1);
+ }
+
+ return 0;
+}
+
+sub do_one_iteration {
+ my ($self) = @_;
+
+ my $haenv = $self->{haenv};
+
+ my $status = $self->get_local_status();
+ my $state = $status->{state};
+
+ # do state changes first
+
+ if ($state eq 'wait_for_quorum') {
+
+ if ($haenv->quorate()) {
+ if ($self->get_protected_ha_manager_lock()) {
+ $self->set_local_status({ state => 'master' });
+ } else {
+ $self->set_local_status({ state => 'slave' });
+ }
+ }
+
+ } elsif ($state eq 'slave') {
+
+ if ($haenv->quorate()) {
+ if ($self->get_protected_ha_manager_lock()) {
+ $self->set_local_status({ state => 'master' });
+ }
+ } else {
+ $self->set_local_status({ state => 'wait_for_quorum' });
+ }
+
+ } elsif ($state eq 'lost_manager_lock') {
+
+ if ($haenv->quorate()) {
+ if ($self->get_protected_ha_manager_lock()) {
+ $self->set_local_status({ state => 'master' });
+ }
+ }
+
+ } elsif ($state eq 'master') {
+
+ if (!$self->get_protected_ha_manager_lock()) {
+ $self->set_local_status({ state => 'lost_manager_lock'});
+ }
+ }
+
+ $status = $self->get_local_status();
+ $state = $status->{state};
+
+ # do work
+
+ if ($state eq 'wait_for_quorum') {
+
+ return 0 if $self->{shutdown_request};
+
+ $haenv->sleep(5);
+
+ } elsif ($state eq 'master') {
+
+ my $manager = $self->{manager};
+
+ die "no manager" if !defined($manager);
+
+ my $startime = $haenv->get_time();
+
+ my $max_time = 10;
+
+ # do work (max_time seconds)
+ eval {
+ # fixme: set alert timer
+ $manager->manage();
+ };
+ if (my $err = $@) {
+ $haenv->log('err', "got unexpected error - $err");
+ }
+
+ $haenv->sleep_until($startime + $max_time);
+
+ } elsif ($state eq 'lost_manager_lock') {
+
+ if ($self->{ha_manager_wd}) {
+ $haenv->watchdog_close($self->{ha_manager_wd});
+ delete $self->{ha_manager_wd};
+ }
+
+ return 0 if $self->{shutdown_request};
+
+ $self->set_local_status({ state => 'wait_for_quorum' });
+
+ } elsif ($state eq 'slave') {
+
+ return 0 if $self->{shutdown_request};
+
+ # wait until we get master
+
+ } else {
+
+ die "got unexpected status '$state'\n";
+ }
+
+ return 1;
+}
+
+1;
--- /dev/null
+package PVE::HA::Env;
+
+use strict;
+use warnings;
+
+use PVE::SafeSyslog;
+use PVE::Tools;
+
+# abstract out the cluster environment for a single node
+
+sub new {
+ my ($this, $baseclass, $node, @args) = @_;
+
+ my $class = ref($this) || $this;
+
+ my $plug = $baseclass->new($node, @args);
+
+ my $self = bless { plug => $plug }, $class;
+
+ return $self;
+}
+
+sub nodename {
+ my ($self) = @_;
+
+ return $self->{plug}->nodename();
+}
+
+# manager status is stored on cluster, protected by ha_manager_lock
+sub read_manager_status {
+ my ($self) = @_;
+
+ return $self->{plug}->read_manager_status();
+}
+
+sub write_manager_status {
+ my ($self, $status_obj) = @_;
+
+ return $self->{plug}->write_manager_status($status_obj);
+}
+
+# lrm status is written by LRM, protected by ha_agent_lock,
+# but can be read by any node (CRM)
+
+sub read_lrm_status {
+ my ($self, $node) = @_;
+
+ return $self->{plug}->read_lrm_status($node);
+}
+
+sub write_lrm_status {
+ my ($self, $status_obj) = @_;
+
+ return $self->{plug}->write_lrm_status($status_obj);
+}
+
+# we use this to enable/disbale ha
+sub manager_status_exists {
+ my ($self) = @_;
+
+ return $self->{plug}->manager_status_exists();
+}
+
+# implement a way to send commands to the CRM master
+sub queue_crm_commands {
+ my ($self, $cmd) = @_;
+
+ return $self->{plug}->queue_crm_commands($cmd);
+}
+
+sub read_crm_commands {
+ my ($self) = @_;
+
+ return $self->{plug}->read_crm_commands();
+}
+
+sub read_service_config {
+ my ($self) = @_;
+
+ return $self->{plug}->read_service_config();
+}
+
+sub change_service_location {
+ my ($self, $sid, $node) = @_;
+
+ return $self->{plug}->change_service_location($sid, $node);
+}
+
+sub read_group_config {
+ my ($self) = @_;
+
+ return $self->{plug}->read_group_config();
+}
+
+# this should return a hash containing info
+# what nodes are members and online.
+sub get_node_info {
+ my ($self) = @_;
+
+ return $self->{plug}->get_node_info();
+}
+
+sub log {
+ my ($self, $level, @args) = @_;
+
+ return $self->{plug}->log($level, @args);
+}
+
+# aquire a cluster wide manager lock
+sub get_ha_manager_lock {
+ my ($self) = @_;
+
+ return $self->{plug}->get_ha_manager_lock();
+}
+
+# aquire a cluster wide node agent lock
+sub get_ha_agent_lock {
+ my ($self) = @_;
+
+ return $self->{plug}->get_ha_agent_lock();
+}
+
+# same as get_ha_agent_lock(), but immeditaley release the lock on success
+sub test_ha_agent_lock {
+ my ($self, $node) = @_;
+
+ return $self->{plug}->test_ha_agent_lock($node);
+}
+
+# return true when cluster is quorate
+sub quorate {
+ my ($self) = @_;
+
+ return $self->{plug}->quorate();
+}
+
+# return current time
+# overwrite that if you want to simulate
+sub get_time {
+ my ($self) = @_;
+
+ return $self->{plug}->get_time();
+}
+
+sub sleep {
+ my ($self, $delay) = @_;
+
+ return $self->{plug}->sleep($delay);
+}
+
+sub sleep_until {
+ my ($self, $end_time) = @_;
+
+ return $self->{plug}->sleep_until($end_time);
+}
+
+sub loop_start_hook {
+ my ($self, @args) = @_;
+
+ return $self->{plug}->loop_start_hook(@args);
+}
+
+sub loop_end_hook {
+ my ($self, @args) = @_;
+
+ return $self->{plug}->loop_end_hook(@args);
+}
+
+sub watchdog_open {
+ my ($self) = @_;
+
+ # Note: when using /dev/watchdog, make sure perl does not close
+ # the handle automatically at exit!!
+
+ return $self->{plug}->watchdog_open();
+}
+
+sub watchdog_update {
+ my ($self, $wfh) = @_;
+
+ return $self->{plug}->watchdog_update($wfh);
+}
+
+sub watchdog_close {
+ my ($self, $wfh) = @_;
+
+ return $self->{plug}->watchdog_close($wfh);
+}
+
+sub exec_resource_agent {
+ my ($self, $sid, $cmd, @params) = @_;
+
+ return $self->{plug}->exec_resource_agent($sid, $cmd, @params)
+}
+
+1;
--- /dev/null
+
+
+.PHONY: install
+install:
+ install -D -m 0644 PVE2.pm ${DESTDIR}${PERLDIR}/PVE/HA/Env/PVE2.pm
--- /dev/null
+package PVE::HA::Env::PVE2;
+
+use strict;
+use warnings;
+use POSIX qw(:errno_h :fcntl_h);
+use IO::File;
+
+use PVE::SafeSyslog;
+use PVE::Tools;
+use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_lock_file);
+
+use PVE::HA::Tools;
+use PVE::HA::Env;
+use PVE::HA::Groups;
+
+my $lockdir = "/etc/pve/priv/lock";
+
+my $manager_status_filename = "/etc/pve/manager_status";
+my $ha_groups_config = "ha/groups.cfg";
+
+#cfs_register_file($ha_groups_config,
+# sub { PVE::HA::Groups->parse_config(@_); },
+# sub { PVE::HA::Groups->write_config(@_); });
+
+sub new {
+ my ($this, $nodename) = @_;
+
+ die "missing nodename" if !$nodename;
+
+ my $class = ref($this) || $this;
+
+ my $self = bless {}, $class;
+
+ $self->{nodename} = $nodename;
+
+ return $self;
+}
+
+sub nodename {
+ my ($self) = @_;
+
+ return $self->{nodename};
+}
+
+sub read_manager_status {
+ my ($self) = @_;
+
+ my $filename = $manager_status_filename;
+
+ return PVE::HA::Tools::read_json_from_file($filename, {});
+}
+
+sub write_manager_status {
+ my ($self, $status_obj) = @_;
+
+ my $filename = $manager_status_filename;
+
+ PVE::HA::Tools::write_json_to_file($filename, $status_obj);
+}
+
+sub read_lrm_status {
+ my ($self, $node) = @_;
+
+ $node = $self->{nodename} if !defined($node);
+
+ my $filename = "/etc/pve/nodes/$node/lrm_status";
+
+ return PVE::HA::Tools::read_json_from_file($filename, {});
+}
+
+sub write_lrm_status {
+ my ($self, $status_obj) = @_;
+
+ my $node = $self->{nodename};
+
+ my $filename = "/etc/pve/nodes/$node/lrm_status";
+
+ PVE::HA::Tools::write_json_to_file($filename, $status_obj);
+}
+
+sub manager_status_exists {
+ my ($self) = @_;
+
+ return -f $manager_status_filename ? 1 : 0;
+}
+
+sub read_service_config {
+ my ($self) = @_;
+
+ die "implement me";
+}
+
+sub change_service_location {
+ my ($self, $sid, $node) = @_;
+
+ die "implement me";
+}
+
+sub read_group_config {
+ my ($self) = @_;
+
+ return cfs_read_file($ha_groups_config);
+}
+
+sub queue_crm_commands {
+ my ($self, $cmd) = @_;
+
+ die "implement me";
+}
+
+sub read_crm_commands {
+ my ($self) = @_;
+
+ die "implement me";
+}
+
+# this should return a hash containing info
+# what nodes are members and online.
+sub get_node_info {
+ my ($self) = @_;
+
+ die "implement me";
+}
+
+sub log {
+ my ($self, $level, $msg) = @_;
+
+ chomp $msg;
+
+ syslog($level, $msg);
+}
+
+my $last_lock_status = {};
+
+sub get_pve_lock {
+ my ($self, $lockid) = @_;
+
+ my $got_lock = 0;
+
+ my $filename = "$lockdir/$lockid";
+
+ my $last = $last_lock_status->{$lockid} || 0;
+
+ my $ctime = time();
+
+ eval {
+
+ mkdir $lockdir;
+
+ # pve cluster filesystem not online
+ die "can't create '$lockdir' (pmxcfs not mounted?)\n" if ! -d $lockdir;
+
+ if ($last && (($ctime - $last) < 100)) { # fixme: what timeout
+ utime(0, $ctime, $filename) || # cfs lock update request
+ die "cfs lock update failed - $!\n";
+ } else {
+
+ # fixme: wait some time?
+ if (!(mkdir $filename)) {
+ utime 0, 0, $filename; # cfs unlock request
+ die "can't get cfs lock\n";
+ }
+ }
+
+ $got_lock = 1;
+ };
+
+ my $err = $@;
+
+ $last_lock_status->{$lockid} = $got_lock ? $ctime : 0;
+
+ if ($got_lock != $last) {
+ if ($got_lock) {
+ $self->log('info', "successfully aquired lock '$lockid'");
+ } else {
+ my $msg = "lost lock '$lockid";
+ $msg .= " - $err" if $err;
+ $self->log('err', $msg);
+ }
+ }
+
+ return $got_lock;
+}
+
+sub get_ha_manager_lock {
+ my ($self) = @_;
+
+ my $lockid = "ha_manager_lock";
+
+ my $filename = "$lockdir/$lockid";
+
+ return $self->get_pve_lock("ha_manager_lock");
+}
+
+sub get_ha_agent_lock {
+ my ($self) = @_;
+
+ my $node = $self->nodename();
+
+ return $self->get_pve_lock("ha_agent_${node}_lock");
+}
+
+sub test_ha_agent_lock {
+ my ($self, $node) = @_;
+
+ my $lockid = "ha_agent_${node}_lock";
+ my $filename = "$lockdir/$lockid";
+ my $res = $self->get_pve_lock($lockid);
+ rmdir $filename if $res; # cfs unlock
+
+ return $res;
+}
+
+sub quorate {
+ my ($self) = @_;
+
+ my $quorate = 0;
+ eval {
+ $quorate = PVE::Cluster::check_cfs_quorum();
+ };
+
+ return $quorate;
+}
+
+sub get_time {
+ my ($self) = @_;
+
+ return time();
+}
+
+sub sleep {
+ my ($self, $delay) = @_;
+
+ CORE::sleep($delay);
+}
+
+sub sleep_until {
+ my ($self, $end_time) = @_;
+
+ for (;;) {
+ my $cur_time = time();
+
+ last if $cur_time >= $end_time;
+
+ $self->sleep(1);
+ }
+}
+
+sub loop_start_hook {
+ my ($self) = @_;
+
+ PVE::Cluster::cfs_update();
+
+ $self->{loop_start} = $self->get_time();
+}
+
+sub loop_end_hook {
+ my ($self) = @_;
+
+ my $delay = $self->get_time() - $self->{loop_start};
+
+ warn "loop take too long ($delay seconds)\n" if $delay > 30;
+}
+
+my $watchdog_fh;
+
+my $WDIOC_GETSUPPORT = 0x80285700;
+my $WDIOC_KEEPALIVE = 0x80045705;
+my $WDIOC_SETTIMEOUT = 0xc0045706;
+my $WDIOC_GETTIMEOUT = 0x80045707;
+
+sub watchdog_open {
+ my ($self) = @_;
+
+ system("modprobe -q softdog soft_noboot=1") if ! -e "/dev/watchdog";
+
+ die "watchdog already open\n" if defined($watchdog_fh);
+
+ $watchdog_fh = IO::File->new(">/dev/watchdog") ||
+ die "unable to open watchdog device - $!\n";
+
+ eval {
+ my $timeoutbuf = pack('I', 100);
+ my $res = ioctl($watchdog_fh, $WDIOC_SETTIMEOUT, $timeoutbuf) ||
+ die "unable to set watchdog timeout - $!\n";
+ my $timeout = unpack("I", $timeoutbuf);
+ die "got wrong watchdog timeout '$timeout'\n" if $timeout != 100;
+
+ my $wdinfo = "\x00" x 40;
+ $res = ioctl($watchdog_fh, $WDIOC_GETSUPPORT, $wdinfo) ||
+ die "unable to get watchdog info - $!\n";
+
+ my ($options, $firmware_version, $indentity) = unpack("lla32", $wdinfo);
+ die "watchdog does not support magic close\n" if !($options & 0x0100);
+
+ };
+ if (my $err = $@) {
+ $self->watchdog_close();
+ die $err;
+ }
+
+ # fixme: use ioctl to setup watchdog timeout (requires C interface)
+
+ $self->log('info', "watchdog active");
+}
+
+sub watchdog_update {
+ my ($self, $wfh) = @_;
+
+ my $res = $watchdog_fh->syswrite("\0", 1);
+ if (!defined($res)) {
+ $self->log('err', "watchdog update failed - $!\n");
+ return 0;
+ }
+ if ($res != 1) {
+ $self->log('err', "watchdog update failed - write $res bytes\n");
+ return 0;
+ }
+
+ return 1;
+}
+
+sub watchdog_close {
+ my ($self, $wfh) = @_;
+
+ $watchdog_fh->syswrite("V", 1); # magic watchdog close
+ if (!$watchdog_fh->close()) {
+ $self->log('err', "watchdog close failed - $!");
+ } else {
+ $watchdog_fh = undef;
+ $self->log('info', "watchdog closed (disabled)");
+ }
+}
+
+sub exec_resource_agent {
+ my ($self, $sid, $cmd, @params) = @_;
+
+ die "implement me";
+}
+
+1;
--- /dev/null
+package PVE::HA::Groups;
+
+use strict;
+use warnings;
+
+use Data::Dumper;
+use PVE::JSONSchema qw(get_standard_option);
+use PVE::SectionConfig;
+
+use base qw(PVE::SectionConfig);
+
+PVE::JSONSchema::register_format('pve-ha-group-node', \&pve_verify_ha_group_node);
+sub pve_verify_ha_group_node {
+ my ($node, $noerr) = @_;
+
+ if ($node !~ m/^([a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?)(:\d+)?$/) {
+ return undef if $noerr;
+ die "value does not look like a valid ha group node\n";
+ }
+ return $node;
+}
+
+PVE::JSONSchema::register_standard_option('pve-ha-group-node-list', {
+ description => "List of cluster node names with optional priority. We use priority '0' as default. The CRM tries to run services on the node with higest priority (also see option 'nofailback').",
+ type => 'string', format => 'pve-ha-group-node-list',
+ typetext => '<node>[:<pri>]{,<node>[:<pri>]}*',
+});
+
+PVE::JSONSchema::register_standard_option('pve-ha-group-id', {
+ description => "The HA group identifier.",
+ type => 'string', format => 'pve-configid',
+});
+
+my $defaultData = {
+ propertyList => {
+ type => { description => "Section type." },
+ group => get_standard_option('pve-ha-group-id'),
+ nodes => get_standard_option('pve-ha-group-node-list'),
+ restricted => {
+ description => "Services on unrestricted groups may run on any cluster members if all group members are offline. But they will migrate back as soon as a group member comes online. One can implement a 'preferred node' behavior using an unrestricted group with one member.",
+ type => 'boolean',
+ optional => 1,
+ default => 0,
+ },
+ nofailback => {
+ description => "The CRM tries to run services on the node with the highest priority. If a node with higher priority comes online, the CRM migrates the service to that node. Enabling nofailback prevents that behavior.",
+ type => 'boolean',
+ optional => 1,
+ default => 0,
+ },
+ comment => {
+ description => "Description.",
+ type => 'string',
+ optional => 1,
+ maxLength => 4096,
+ },
+ },
+};
+
+sub type {
+ return 'group';
+}
+
+sub options {
+ return {
+ nodes => {},
+ comment => { optional => 1 },
+ };
+}
+
+sub private {
+ return $defaultData;
+}
+
+sub parse_section_header {
+ my ($class, $line) = @_;
+
+ if ($line =~ m/^(\S+):\s*(\S+)\s*$/) {
+ my ($type, $group) = (lc($1), $2);
+ my $errmsg = undef; # set if you want to skip whole section
+ eval { PVE::JSONSchema::pve_verify_configid($group); };
+ $errmsg = $@ if $@;
+ my $config = {}; # to return additional attributes
+ return ($type, $group, $errmsg, $config);
+ }
+ return undef;
+}
+
+1;
--- /dev/null
+package PVE::HA::LRM;
+
+# Local Resource Manager
+
+use strict;
+use warnings;
+use Data::Dumper;
+use POSIX qw(:sys_wait_h);
+
+use PVE::SafeSyslog;
+use PVE::Tools;
+use PVE::HA::Tools;
+
+# Server can have several states:
+
+my $valid_states = {
+ wait_for_agent_lock => "waiting for agnet lock",
+ active => "got agent_lock",
+ lost_agent_lock => "lost agent_lock",
+};
+
+sub new {
+ my ($this, $haenv) = @_;
+
+ my $class = ref($this) || $this;
+
+ my $self = bless {
+ haenv => $haenv,
+ status => { state => 'startup' },
+ workers => {},
+ results => {},
+ }, $class;
+
+ $self->set_local_status({ state => 'wait_for_agent_lock' });
+
+ return $self;
+}
+
+sub shutdown_request {
+ my ($self) = @_;
+
+ $self->{shutdown_request} = 1;
+}
+
+sub get_local_status {
+ my ($self) = @_;
+
+ return $self->{status};
+}
+
+sub set_local_status {
+ my ($self, $new) = @_;
+
+ die "invalid state '$new->{state}'" if !$valid_states->{$new->{state}};
+
+ my $haenv = $self->{haenv};
+
+ my $old = $self->{status};
+
+ # important: only update if if really changed
+ return if $old->{state} eq $new->{state};
+
+ $haenv->log('info', "status change $old->{state} => $new->{state}");
+
+ $new->{state_change_time} = $haenv->get_time();
+
+ $self->{status} = $new;
+}
+
+sub get_protected_ha_agent_lock {
+ my ($self) = @_;
+
+ my $haenv = $self->{haenv};
+
+ my $count = 0;
+ my $starttime = $haenv->get_time();
+
+ for (;;) {
+
+ if ($haenv->get_ha_agent_lock()) {
+ if ($self->{ha_agent_wd}) {
+ $haenv->watchdog_update($self->{ha_agent_wd});
+ } else {
+ my $wfh = $haenv->watchdog_open();
+ $self->{ha_agent_wd} = $wfh;
+ }
+ return 1;
+ }
+
+ last if ++$count > 5; # try max 5 time
+
+ my $delay = $haenv->get_time() - $starttime;
+ last if $delay > 5; # for max 5 seconds
+
+ $haenv->sleep(1);
+ }
+
+ return 0;
+}
+
+sub do_one_iteration {
+ my ($self) = @_;
+
+ my $haenv = $self->{haenv};
+
+ my $status = $self->get_local_status();
+ my $state = $status->{state};
+
+ # do state changes first
+
+ my $ctime = $haenv->get_time();
+
+ if ($state eq 'wait_for_agent_lock') {
+
+ my $service_count = 1; # todo: correctly compute
+
+ if ($service_count && $haenv->quorate()) {
+ if ($self->get_protected_ha_agent_lock()) {
+ $self->set_local_status({ state => 'active' });
+ }
+ }
+
+ } elsif ($state eq 'lost_agent_lock') {
+
+ if ($haenv->quorate()) {
+ if ($self->get_protected_ha_agent_lock()) {
+ $self->set_local_status({ state => 'active' });
+ }
+ }
+
+ } elsif ($state eq 'active') {
+
+ if (!$self->get_protected_ha_agent_lock()) {
+ $self->set_local_status({ state => 'lost_agent_lock'});
+ }
+ }
+
+ $status = $self->get_local_status();
+ $state = $status->{state};
+
+ # do work
+
+ $self->{service_status} = {};
+
+ if ($state eq 'wait_for_agent_lock') {
+
+ return 0 if $self->{shutdown_request};
+
+ $haenv->sleep(5);
+
+ } elsif ($state eq 'active') {
+
+ my $startime = $haenv->get_time();
+
+ my $max_time = 10;
+
+ my $shutdown = 0;
+
+ # do work (max_time seconds)
+ eval {
+ # fixme: set alert timer
+
+ if ($self->{shutdown_request}) {
+
+ # fixme: request service stop or relocate ?
+
+ my $service_count = 0; # fixme
+
+ if ($service_count == 0) {
+
+ if ($self->{ha_agent_wd}) {
+ $haenv->watchdog_close($self->{ha_agent_wd});
+ delete $self->{ha_agent_wd};
+ }
+
+ $shutdown = 1;
+ }
+ } else {
+ my $ms = $haenv->read_manager_status();
+
+ $self->{service_status} = $ms->{service_status} || {};
+
+ $self->manage_resources();
+ }
+ };
+ if (my $err = $@) {
+ $haenv->log('err', "got unexpected error - $err");
+ }
+
+ return 0 if $shutdown;
+
+ $haenv->sleep_until($startime + $max_time);
+
+ } elsif ($state eq 'lost_agent_lock') {
+
+ # Note: watchdog is active an will triger soon!
+
+ # so we hope to get the lock back soon!
+
+ if ($self->{shutdown_request}) {
+
+ my $running_services = 0; # fixme: correctly compute
+
+ if ($running_services > 0) {
+ $haenv->log('err', "get shutdown request in state 'lost_agent_lock' - " .
+ "killing running services");
+
+ # fixme: kill all services as fast as possible
+ }
+
+ # now all services are stopped, so we can close the watchdog
+
+ if ($self->{ha_agent_wd}) {
+ $haenv->watchdog_close($self->{ha_agent_wd});
+ delete $self->{ha_agent_wd};
+ }
+
+ return 0;
+ }
+
+ } else {
+
+ die "got unexpected status '$state'\n";
+
+ }
+
+ return 1;
+}
+
+sub manage_resources {
+ my ($self) = @_;
+
+ my $haenv = $self->{haenv};
+
+ my $nodename = $haenv->nodename();
+
+ my $ms = $haenv->read_manager_status();
+
+ my $ss = $self->{service_status};
+
+ foreach my $sid (keys %$ss) {
+ my $sd = $ss->{$sid};
+ next if !$sd->{node};
+ next if !$sd->{uid};
+ next if $sd->{node} ne $nodename;
+ my $req_state = $sd->{state};
+ next if !defined($req_state);
+
+ eval {
+ $self->queue_resource_command($sid, $sd->{uid}, $req_state, $sd->{target});
+ };
+ if (my $err = $@) {
+ warn "unable to run resource agent for '$sid' - $err"; # fixme
+ }
+ }
+
+ my $starttime = time();
+
+ # start workers
+ my $max_workers = 4;
+
+ while ((time() - $starttime) < 5) {
+ my $count = $self->check_active_workers();
+
+ foreach my $sid (keys %{$self->{workers}}) {
+ last if $count >= $max_workers;
+ my $w = $self->{workers}->{$sid};
+ if (!$w->{pid}) {
+ my $pid = fork();
+ if (!defined($pid)) {
+ warn "fork worker failed\n";
+ $count = 0; last; # abort, try later
+ } elsif ($pid == 0) {
+ # do work
+ my $res = -1;
+ eval {
+ $res = $haenv->exec_resource_agent($sid, $w->{state}, $w->{target});
+ };
+ if (my $err = $@) {
+ warn $err;
+ POSIX::_exit(-1);
+ }
+ POSIX::_exit($res);
+ } else {
+ $count++;
+ $w->{pid} = $pid;
+ }
+ }
+ }
+
+ last if !$count;
+
+ sleep(1);
+ }
+}
+
+# fixme: use a queue an limit number of parallel workers?
+sub queue_resource_command {
+ my ($self, $sid, $uid, $state, $target) = @_;
+
+ if (my $w = $self->{workers}->{$sid}) {
+ return if $w->{pid}; # already started
+ # else, delete and overwrite queue entry with new command
+ delete $self->{workers}->{$sid};
+ }
+
+ $self->{workers}->{$sid} = {
+ sid => $sid,
+ uid => $uid,
+ state => $state,
+ };
+
+ $self->{workers}->{$sid}->{target} = $target if $target;
+}
+
+sub check_active_workers {
+ my ($self) = @_;
+
+ # finish/count workers
+ my $count = 0;
+ foreach my $sid (keys %{$self->{workers}}) {
+ my $w = $self->{workers}->{$sid};
+ if (my $pid = $w->{pid}) {
+ # check status
+ my $waitpid = waitpid($pid, WNOHANG);
+ if (defined($waitpid) && ($waitpid == $pid)) {
+ $self->resource_command_finished($sid, $w->{uid}, $?);
+ } else {
+ $count++;
+ }
+ }
+ }
+
+ return $count;
+}
+
+sub resource_command_finished {
+ my ($self, $sid, $uid, $status) = @_;
+
+ my $haenv = $self->{haenv};
+
+ my $w = delete $self->{workers}->{$sid};
+ return if !$w; # should not happen
+
+ my $exit_code = -1;
+
+ if ($status == -1) {
+ $haenv->log('err', "resource agent $sid finished - failed to execute");
+ } elsif (my $sig = ($status & 127)) {
+ $haenv->log('err', "resource agent $sid finished - got signal $sig");
+ } else {
+ $exit_code = ($status >> 8);
+ }
+
+ $self->{results}->{$uid} = {
+ sid => $w->{sid},
+ state => $w->{state},
+ exit_code => $exit_code,
+ };
+
+ my $ss = $self->{service_status};
+
+ # compute hash of valid/existing uids
+ my $valid_uids = {};
+ foreach my $sid (keys %$ss) {
+ my $sd = $ss->{$sid};
+ next if !$sd->{uid};
+ $valid_uids->{$sd->{uid}} = 1;
+ }
+
+ my $results = {};
+ foreach my $id (keys %{$self->{results}}) {
+ next if !$valid_uids->{$id};
+ $results->{$id} = $self->{results}->{$id};
+ }
+ $self->{results} = $results;
+
+ $haenv->write_lrm_status($results);
+}
+
+1;
--- /dev/null
+SOURCES=CRM.pm Env.pm Groups.pm LRM.pm Manager.pm NodeStatus.pm Tools.pm
+
+.PHONY: install
+install:
+ install -d -m 0755 ${DESTDIR}${PERLDIR}/PVE/HA
+ for i in ${SOURCES}; do install -D -m 0644 $$i ${DESTDIR}${PERLDIR}/PVE/HA/$$i; done
+ make -C Env install
+
+.PHONY: installsim
+installsim:
+ install -d -m 0755 ${DESTDIR}${PERLDIR}/PVE/HA
+ for i in ${SOURCES}; do install -D -m 0644 $$i ${DESTDIR}${PERLDIR}/PVE/HA/$$i; done
+ make -C Sim installsim
--- /dev/null
+package PVE::HA::Manager;
+
+use strict;
+use warnings;
+use Digest::MD5 qw(md5_base64);
+
+use Data::Dumper;
+
+use PVE::HA::NodeStatus;
+
+sub new {
+ my ($this, $haenv) = @_;
+
+ my $class = ref($this) || $this;
+
+ my $ms = $haenv->read_manager_status();
+
+ $ms->{master_node} = $haenv->nodename();
+
+ my $ns = PVE::HA::NodeStatus->new($haenv, $ms->{node_status} || {});
+
+ # fixme: use separate class PVE::HA::ServiceStatus
+ my $ss = $ms->{service_status} || {};
+
+ my $self = bless {
+ haenv => $haenv,
+ ms => $ms, # master status
+ ns => $ns, # PVE::HA::NodeStatus
+ ss => $ss, # service status
+ }, $class;
+
+ return $self;
+}
+
+sub cleanup {
+ my ($self) = @_;
+
+ # todo: ?
+}
+
+sub flush_master_status {
+ my ($self) = @_;
+
+ my ($haenv, $ms, $ns, $ss) = ($self->{haenv}, $self->{ms}, $self->{ns}, $self->{ss});
+
+ $ms->{node_status} = $ns->{status};
+ $ms->{service_status} = $ss;
+
+ $haenv->write_manager_status($ms);
+}
+
+sub select_service_node {
+ my ($groups, $online_node_usage, $service_conf, $current_node, $try_next) = @_;
+
+ my $group = { 'nodes' => $service_conf->{node} }; # default group
+
+ $group = $groups->{ids}->{$service_conf->{group}} if $service_conf->{group} &&
+ $groups->{ids}->{$service_conf->{group}};
+
+ my $pri_groups = {};
+ my $group_members = {};
+ foreach my $entry (PVE::Tools::split_list($group->{nodes})) {
+ my ($node, $pri) = ($entry, 0);
+ if ($entry =~ m/^(\S+):(\d+)$/) {
+ ($node, $pri) = ($1, $2);
+ }
+ next if !defined($online_node_usage->{$node}); # offline
+ $pri_groups->{$pri}->{$node} = 1;
+ $group_members->{$node} = $pri;
+ }
+
+
+ # add non-group members to unrestricted groups (priority -1)
+ if (!$group->{restricted}) {
+ my $pri = -1;
+ foreach my $node (keys %$online_node_usage) {
+ next if defined($group_members->{$node});
+ $pri_groups->{$pri}->{$node} = 1;
+ $group_members->{$node} = -1;
+ }
+ }
+
+
+ my @pri_list = sort {$b <=> $a} keys %$pri_groups;
+ return undef if !scalar(@pri_list);
+
+ if (!$try_next && $group->{nofailback} && defined($group_members->{$current_node})) {
+ return $current_node;
+ }
+
+ # select node from top priority node list
+
+ my $top_pri = $pri_list[0];
+
+ my @nodes = sort { $online_node_usage->{$a} <=> $online_node_usage->{$b} } keys %{$pri_groups->{$top_pri}};
+
+ my $found;
+ for (my $i = scalar(@nodes) - 1; $i >= 0; $i--) {
+ my $node = $nodes[$i];
+ if ($node eq $current_node) {
+ $found = $i;
+ last;
+ }
+ }
+
+ if ($try_next) {
+
+ if (defined($found) && ($found < (scalar(@nodes) - 1))) {
+ return $nodes[$found + 1];
+ } else {
+ return $nodes[0];
+ }
+
+ } else {
+
+ return $nodes[$found] if defined($found);
+
+ return $nodes[0];
+
+ }
+}
+
+my $uid_counter = 0;
+
+my $valid_service_states = {
+ stopped => 1,
+ request_stop => 1,
+ started => 1,
+ fence => 1,
+ migrate => 1,
+ relocate => 1,
+ error => 1,
+};
+
+sub recompute_online_node_usage {
+ my ($self) = @_;
+
+ my $online_node_usage = {};
+
+ my $online_nodes = $self->{ns}->list_online_nodes();
+
+ foreach my $node (@$online_nodes) {
+ $online_node_usage->{$node} = 0;
+ }
+
+ foreach my $sid (keys %{$self->{ss}}) {
+ my $sd = $self->{ss}->{$sid};
+ my $state = $sd->{state};
+ if (defined($online_node_usage->{$sd->{node}})) {
+ if (($state eq 'started') || ($state eq 'request_stop') ||
+ ($state eq 'fence') || ($state eq 'error')) {
+ $online_node_usage->{$sd->{node}}++;
+ } elsif (($state eq 'migrate') || ($state eq 'relocate')) {
+ $online_node_usage->{$sd->{target}}++;
+ } elsif ($state eq 'stopped') {
+ # do nothing
+ } else {
+ die "should not be reached";
+ }
+ }
+ }
+
+ $self->{online_node_usage} = $online_node_usage;
+}
+
+my $change_service_state = sub {
+ my ($self, $sid, $new_state, %params) = @_;
+
+ my ($haenv, $ss) = ($self->{haenv}, $self->{ss});
+
+ my $sd = $ss->{$sid} || die "no such service '$sid";
+
+ my $old_state = $sd->{state};
+ my $old_node = $sd->{node};
+
+ die "no state change" if $old_state eq $new_state; # just to be sure
+
+ die "invalid CRM service state '$new_state'\n" if !$valid_service_states->{$new_state};
+
+ foreach my $k (keys %$sd) { delete $sd->{$k}; };
+
+ $sd->{state} = $new_state;
+ $sd->{node} = $old_node;
+
+ my $text_state = '';
+ foreach my $k (keys %params) {
+ my $v = $params{$k};
+ $text_state .= ", " if $text_state;
+ $text_state .= "$k = $v";
+ $sd->{$k} = $v;
+ }
+
+ $self->recompute_online_node_usage();
+
+ $uid_counter++;
+ $sd->{uid} = md5_base64($new_state . $$ . time() . $uid_counter);
+
+ $text_state = " ($text_state)" if $text_state;
+ $haenv->log('info', "service '$sid': state changed from '${old_state}' to '${new_state}' $text_state\n");
+};
+
+# read LRM status for all active nodes
+sub read_lrm_status {
+ my ($self) = @_;
+
+ my $nodes = $self->{ns}->list_online_nodes();
+ my $haenv = $self->{haenv};
+
+ my $res = {};
+
+ foreach my $node (@$nodes) {
+ my $ls = $haenv->read_lrm_status($node);
+ foreach my $uid (keys %$ls) {
+ next if $res->{$uid}; # should not happen
+ $res->{$uid} = $ls->{$uid};
+ }
+ }
+
+ return $res;
+}
+
+# read new crm commands and save them into crm master status
+sub update_crm_commands {
+ my ($self) = @_;
+
+ my ($haenv, $ms, $ns, $ss) = ($self->{haenv}, $self->{ms}, $self->{ns}, $self->{ss});
+
+ my $cmdlist = $haenv->read_crm_commands();
+
+ foreach my $cmd (split(/\n/, $cmdlist)) {
+ chomp $cmd;
+
+ if ($cmd =~ m/^(migrate|relocate)\s+(\S+)\s+(\S+)$/) {
+ my ($task, $sid, $node) = ($1, $2, $3);
+ if (my $sd = $ss->{$sid}) {
+ if (!$ns->node_is_online($node)) {
+ $haenv->log('err', "crm command error - node not online: $cmd");
+ } else {
+ if ($node eq $sd->{node}) {
+ $haenv->log('info', "ignore crm command - service already on target node: $cmd");
+ } else {
+ $haenv->log('info', "got crm command: $cmd");
+ $ss->{$sid}->{cmd} = [ $task, $node];
+ }
+ }
+ } else {
+ $haenv->log('err', "crm command error - no such service: $cmd");
+ }
+
+ } else {
+ $haenv->log('err', "unable to parse crm command: $cmd");
+ }
+ }
+
+}
+
+sub manage {
+ my ($self) = @_;
+
+ my ($haenv, $ms, $ns, $ss) = ($self->{haenv}, $self->{ms}, $self->{ns}, $self->{ss});
+
+ $ns->update($haenv->get_node_info());
+
+ if (!$ns->node_is_online($haenv->nodename())) {
+ $haenv->log('info', "master seems offline\n");
+ return;
+ }
+
+ my $lrm_status = $self->read_lrm_status();
+
+ my $sc = $haenv->read_service_config();
+
+ $self->{groups} = $haenv->read_group_config(); # update
+
+ # compute new service status
+
+ # add new service
+ foreach my $sid (keys %$sc) {
+ next if $ss->{$sid}; # already there
+ $haenv->log('info', "Adding new service '$sid'\n");
+ # assume we are running to avoid relocate running service at add
+ $ss->{$sid} = { state => 'started', node => $sc->{$sid}->{node}};
+ }
+
+ $self->update_crm_commands();
+
+ for (;;) {
+ my $repeat = 0;
+
+ $self->recompute_online_node_usage();
+
+ foreach my $sid (keys %$ss) {
+ my $sd = $ss->{$sid};
+ my $cd = $sc->{$sid} || { state => 'disabled' };
+
+ my $lrm_res = $sd->{uid} ? $lrm_status->{$sd->{uid}} : undef;
+
+ my $last_state = $sd->{state};
+
+ if ($last_state eq 'stopped') {
+
+ $self->next_state_stopped($sid, $cd, $sd, $lrm_res);
+
+ } elsif ($last_state eq 'started') {
+
+ $self->next_state_started($sid, $cd, $sd, $lrm_res);
+
+ } elsif ($last_state eq 'migrate' || $last_state eq 'relocate') {
+
+ $self->next_state_migrate_relocate($sid, $cd, $sd, $lrm_res);
+
+ } elsif ($last_state eq 'fence') {
+
+ # do nothing here - wait until fenced
+
+ } elsif ($last_state eq 'request_stop') {
+
+ $self->next_state_request_stop($sid, $cd, $sd, $lrm_res);
+
+ } elsif ($last_state eq 'error') {
+
+ # fixme:
+
+ } else {
+
+ die "unknown service state '$last_state'";
+ }
+
+ $repeat = 1 if $sd->{state} ne $last_state;
+ }
+
+ # handle fencing
+ my $fenced_nodes = {};
+ foreach my $sid (keys %$ss) {
+ my $sd = $ss->{$sid};
+ next if $sd->{state} ne 'fence';
+
+ if (!defined($fenced_nodes->{$sd->{node}})) {
+ $fenced_nodes->{$sd->{node}} = $ns->fence_node($sd->{node}) || 0;
+ }
+
+ next if !$fenced_nodes->{$sd->{node}};
+
+ # node fence was sucessful - mark service as stopped
+ &$change_service_state($self, $sid, 'stopped');
+ }
+
+ last if !$repeat;
+ }
+
+ # remove stale services
+ # fixme:
+
+ $self->flush_master_status();
+}
+
+# functions to compute next service states
+# $cd: service configuration data (read only)
+# $sd: service status data (read only)
+#
+# Note: use change_service_state() to alter state
+#
+
+sub next_state_request_stop {
+ my ($self, $sid, $cd, $sd, $lrm_res) = @_;
+
+ my $haenv = $self->{haenv};
+ my $ns = $self->{ns};
+
+ # check result from LRM daemon
+ if ($lrm_res) {
+ my $exit_code = $lrm_res->{exit_code};
+ if ($exit_code == 0) {
+ &$change_service_state($self, $sid, 'stopped');
+ return;
+ } else {
+ &$change_service_state($self, $sid, 'error'); # fixme: what state?
+ return;
+ }
+ }
+
+ if (!$ns->node_is_online($sd->{node})) {
+ &$change_service_state($self, $sid, 'fence');
+ return;
+ }
+}
+
+sub next_state_migrate_relocate {
+ my ($self, $sid, $cd, $sd, $lrm_res) = @_;
+
+ my $haenv = $self->{haenv};
+ my $ns = $self->{ns};
+
+ # check result from LRM daemon
+ if ($lrm_res) {
+ my $exit_code = $lrm_res->{exit_code};
+ if ($exit_code == 0) {
+ &$change_service_state($self, $sid, 'started', node => $sd->{target});
+ return;
+ } else {
+ $haenv->log('err', "service '$sid' - migration failed (exit code $exit_code)");
+ &$change_service_state($self, $sid, 'started', node => $sd->{node});
+ return;
+ }
+ }
+
+ if (!$ns->node_is_online($sd->{node})) {
+ &$change_service_state($self, $sid, 'fence');
+ return;
+ }
+}
+
+
+sub next_state_stopped {
+ my ($self, $sid, $cd, $sd, $lrm_res) = @_;
+
+ my $haenv = $self->{haenv};
+ my $ns = $self->{ns};
+
+ if ($sd->{node} ne $cd->{node}) {
+ # this can happen if we fence a node with active migrations
+ # hack: modify $sd (normally this should be considered read-only)
+ $haenv->log('info', "fixup service '$sid' location ($sd->{node} => $cd->{node}");
+ $sd->{node} = $cd->{node};
+ }
+
+ if ($sd->{cmd}) {
+ my ($cmd, $target) = @{$sd->{cmd}};
+ delete $sd->{cmd};
+
+ if ($cmd eq 'migrate' || $cmd eq 'relocate') {
+ if (!$ns->node_is_online($target)) {
+ $haenv->log('err', "ignore service '$sid' $cmd request - node '$target' not online");
+ } elsif ($sd->{node} eq $target) {
+ $haenv->log('info', "ignore service '$sid' $cmd request - service already on node '$target'");
+ } else {
+ $haenv->change_service_location($sid, $target);
+ $cd->{node} = $sd->{node} = $target; # fixme: $sd is read-only??!!
+ $haenv->log('info', "$cmd service '$sid' to node '$target' (stopped)");
+ }
+ } else {
+ $haenv->log('err', "unknown command '$cmd' for service '$sid'");
+ }
+ }
+
+ if ($cd->{state} eq 'disabled') {
+ # do nothing
+ return;
+ }
+
+ if ($cd->{state} eq 'enabled') {
+ if (my $node = select_service_node($self->{groups}, $self->{online_node_usage}, $cd, $sd->{node})) {
+ if ($node && ($sd->{node} ne $node)) {
+ $haenv->change_service_location($sid, $node);
+ }
+ &$change_service_state($self, $sid, 'started', node => $node);
+ } else {
+ # fixme: warn
+ }
+
+ return;
+ }
+
+ $haenv->log('err', "service '$sid' - unknown state '$cd->{state}' in service configuration");
+}
+
+sub next_state_started {
+ my ($self, $sid, $cd, $sd, $lrm_res) = @_;
+
+ my $haenv = $self->{haenv};
+ my $ns = $self->{ns};
+
+ if (!$ns->node_is_online($sd->{node})) {
+
+ &$change_service_state($self, $sid, 'fence');
+ return;
+ }
+
+ if ($cd->{state} eq 'disabled') {
+ &$change_service_state($self, $sid, 'request_stop');
+ return;
+ }
+
+ if ($cd->{state} eq 'enabled') {
+
+ if ($sd->{cmd}) {
+ my ($cmd, $target) = @{$sd->{cmd}};
+ delete $sd->{cmd};
+
+ if ($cmd eq 'migrate' || $cmd eq 'relocate') {
+ if (!$ns->node_is_online($target)) {
+ $haenv->log('err', "ignore service '$sid' $cmd request - node '$target' not online");
+ } elsif ($sd->{node} eq $target) {
+ $haenv->log('info', "ignore service '$sid' $cmd request - service already on node '$target'");
+ } else {
+ $haenv->log('info', "$cmd service '$sid' to node '$target' (running)");
+ &$change_service_state($self, $sid, $cmd, node => $sd->{node}, target => $target);
+ }
+ } else {
+ $haenv->log('err', "unknown command '$cmd' for service '$sid'");
+ }
+ } else {
+
+ my $try_next = 0;
+ if ($lrm_res && ($lrm_res->{exit_code} != 0)) { # fixme: other exit codes?
+ $try_next = 1;
+ }
+
+ my $node = select_service_node($self->{groups}, $self->{online_node_usage},
+ $cd, $sd->{node}, $try_next);
+
+ if ($node && ($sd->{node} ne $node)) {
+ $haenv->log('info', "migrate service '$sid' to node '$node' (running)");
+ &$change_service_state($self, $sid, 'migrate', node => $sd->{node}, target => $node);
+ } else {
+ # do nothing
+ }
+ }
+
+ return;
+ }
+
+ $haenv->log('err', "service '$sid' - unknown state '$cd->{state}' in service configuration");
+}
+
+1;
--- /dev/null
+package PVE::HA::NodeStatus;
+
+use strict;
+use warnings;
+
+use Data::Dumper;
+
+sub new {
+ my ($this, $haenv, $status) = @_;
+
+ my $class = ref($this) || $this;
+
+ my $self = bless {
+ haenv => $haenv,
+ status => $status,
+ }, $class;
+
+ return $self;
+}
+
+# possible node state:
+my $valid_node_states = {
+ online => "node online and member of quorate partition",
+ unknown => "not member of quorate partition, but possibly still running",
+ fence => "node needs to be fenced",
+};
+
+sub get_node_state {
+ my ($self, $node) = @_;
+
+ $self->{status}->{$node} = 'unknown'
+ if !$self->{status}->{$node};
+
+ return $self->{status}->{$node};
+}
+
+sub node_is_online {
+ my ($self, $node) = @_;
+
+ return $self->get_node_state($node) eq 'online';
+}
+
+sub list_online_nodes {
+ my ($self) = @_;
+
+ my $res = [];
+
+ foreach my $node (sort keys %{$self->{status}}) {
+ next if $self->{status}->{$node} ne 'online';
+ push @$res, $node;
+ }
+
+ return $res;
+}
+
+my $set_node_state = sub {
+ my ($self, $node, $state) = @_;
+
+ my $haenv = $self->{haenv};
+
+ die "unknown node state '$state'\n"
+ if !defined($valid_node_states->{$state});
+
+ my $last_state = $self->get_node_state($node);
+
+ return if $state eq $last_state;
+
+ $self->{status}->{$node} = $state;
+
+ $haenv->log('info', "node '$node': state changed from " .
+ "'$last_state' => '$state'\n");
+
+};
+
+sub update {
+ my ($self, $node_info) = @_;
+
+ foreach my $node (keys %$node_info) {
+ my $d = $node_info->{$node};
+ next if !$d->{online};
+
+ my $state = $self->get_node_state($node);
+
+ if ($state eq 'online') {
+ # &$set_node_state($self, $node, 'online');
+ } elsif ($state eq 'unknown') {
+ &$set_node_state($self, $node, 'online');
+ } elsif ($state eq 'fence') {
+ # do nothing, wait until fenced
+ } else {
+ die "detected unknown node state '$state";
+ }
+ }
+
+ foreach my $node (keys %{$self->{status}}) {
+ my $d = $node_info->{$node};
+ next if $d && $d->{online};
+
+ my $state = $self->get_node_state($node);
+
+ # node is not inside quorate partition, possibly not active
+
+ if ($state eq 'online') {
+ &$set_node_state($self, $node, 'unknown');
+ } elsif ($state eq 'unknown') {
+ # &$set_node_state($self, $node, 'unknown');
+ } elsif ($state eq 'fence') {
+ # do nothing, wait until fenced
+ } else {
+ die "detected unknown node state '$state";
+ }
+
+ }
+}
+
+# start fencing
+sub fence_node {
+ my ($self, $node) = @_;
+
+ my $haenv = $self->{haenv};
+
+ my $state = $self->get_node_state($node);
+
+ if ($state ne 'fence') {
+ &$set_node_state($self, $node, 'fence');
+ }
+
+ my $success = $haenv->test_ha_agent_lock($node);
+
+ if ($success) {
+ $haenv->log("info", "fencing: acknowleged - got agent lock for node '$node'");
+ &$set_node_state($self, $node, 'unknown');
+ }
+
+ return $success;
+}
+
+1;
--- /dev/null
+package PVE::HA::Sim::Env;
+
+use strict;
+use warnings;
+use POSIX qw(strftime EINTR);
+use Data::Dumper;
+use JSON;
+use IO::File;
+use Fcntl qw(:DEFAULT :flock);
+
+use PVE::HA::Tools;
+use PVE::HA::Env;
+
+sub new {
+ my ($this, $nodename, $hardware, $log_id) = @_;
+
+ die "missing nodename" if !$nodename;
+ die "missing log_id" if !$log_id;
+
+ my $class = ref($this) || $this;
+
+ my $self = bless {}, $class;
+
+ $self->{statusdir} = $hardware->statusdir();
+ $self->{nodename} = $nodename;
+
+ $self->{hardware} = $hardware;
+ $self->{lock_timeout} = 120;
+
+ $self->{log_id} = $log_id;
+
+ return $self;
+}
+
+sub nodename {
+ my ($self) = @_;
+
+ return $self->{nodename};
+}
+
+sub sim_get_lock {
+ my ($self, $lock_name, $unlock) = @_;
+
+ return 0 if !$self->quorate();
+
+ my $filename = "$self->{statusdir}/cluster_locks";
+
+ my $code = sub {
+
+ my $data = PVE::HA::Tools::read_json_from_file($filename, {});
+
+ my $res;
+
+ my $nodename = $self->nodename();
+ my $ctime = $self->get_time();
+
+ if ($unlock) {
+
+ if (my $d = $data->{$lock_name}) {
+ my $tdiff = $ctime - $d->{time};
+
+ if ($tdiff > $self->{lock_timeout}) {
+ $res = 1;
+ } elsif (($tdiff <= $self->{lock_timeout}) && ($d->{node} eq $nodename)) {
+ delete $data->{$lock_name};
+ $res = 1;
+ } else {
+ $res = 0;
+ }
+ }
+
+ } else {
+
+ if (my $d = $data->{$lock_name}) {
+
+ my $tdiff = $ctime - $d->{time};
+
+ if ($tdiff <= $self->{lock_timeout}) {
+ if ($d->{node} eq $nodename) {
+ $d->{time} = $ctime;
+ $res = 1;
+ } else {
+ $res = 0;
+ }
+ } else {
+ $self->log('info', "got lock '$lock_name'");
+ $d->{node} = $nodename;
+ $d->{time} = $ctime;
+ $res = 1;
+ }
+
+ } else {
+ $data->{$lock_name} = {
+ time => $ctime,
+ node => $nodename,
+ };
+ $self->log('info', "got lock '$lock_name'");
+ $res = 1;
+ }
+ }
+
+ PVE::HA::Tools::write_json_to_file($filename, $data);
+
+ return $res;
+ };
+
+ return $self->{hardware}->global_lock($code);
+}
+
+sub read_manager_status {
+ my ($self) = @_;
+
+ my $filename = "$self->{statusdir}/manager_status";
+
+ return PVE::HA::Tools::read_json_from_file($filename, {});
+}
+
+sub write_manager_status {
+ my ($self, $status_obj) = @_;
+
+ my $filename = "$self->{statusdir}/manager_status";
+
+ PVE::HA::Tools::write_json_to_file($filename, $status_obj);
+}
+
+sub manager_status_exists {
+ my ($self) = @_;
+
+ my $filename = "$self->{statusdir}/manager_status";
+
+ return -f $filename ? 1 : 0;
+}
+
+sub read_lrm_status {
+ my ($self, $node) = @_;
+
+ $node = $self->{nodename} if !defined($node);
+
+ return $self->{hardware}->read_lrm_status($node);
+}
+
+sub write_lrm_status {
+ my ($self, $status_obj) = @_;
+
+ my $node = $self->{nodename};
+
+ return $self->{hardware}->write_lrm_status($node, $status_obj);
+}
+
+sub read_service_config {
+ my ($self) = @_;
+
+ return $self->{hardware}->read_service_config();
+}
+
+sub read_group_config {
+ my ($self) = @_;
+
+ return $self->{hardware}->read_group_config();
+}
+
+sub change_service_location {
+ my ($self, $sid, $node) = @_;
+
+ return $self->{hardware}->change_service_location($sid, $node);
+}
+
+sub queue_crm_commands {
+ my ($self, $cmd) = @_;
+
+ return $self->{hardware}->queue_crm_commands($cmd);
+}
+
+sub read_crm_commands {
+ my ($self) = @_;
+
+ return $self->{hardware}->read_crm_commands();
+}
+
+sub log {
+ my ($self, $level, $msg) = @_;
+
+ chomp $msg;
+
+ my $time = $self->get_time();
+
+ printf("%-5s %5d %12s: $msg\n", $level, $time, "$self->{nodename}/$self->{log_id}");
+}
+
+sub get_time {
+ my ($self) = @_;
+
+ die "implement in subclass";
+}
+
+sub sleep {
+ my ($self, $delay) = @_;
+
+ die "implement in subclass";
+}
+
+sub sleep_until {
+ my ($self, $end_time) = @_;
+
+ die "implement in subclass";
+}
+
+sub get_ha_manager_lock {
+ my ($self) = @_;
+
+ return $self->sim_get_lock('ha_manager_lock');
+}
+
+sub get_ha_agent_lock_name {
+ my ($self, $node) = @_;
+
+ $node = $self->nodename() if !$node;
+
+ return "ha_agent_${node}_lock";
+}
+
+sub get_ha_agent_lock {
+ my ($self) = @_;
+
+ my $lck = $self->get_ha_agent_lock_name();
+ return $self->sim_get_lock($lck);
+}
+
+sub test_ha_agent_lock {
+ my ($self, $node) = @_;
+
+ my $lck = $self->get_ha_agent_lock_name($node);
+ my $res = $self->sim_get_lock($lck);
+ $self->sim_get_lock($lck, 1) if $res; # unlock
+ return $res;
+}
+
+# return true when cluster is quorate
+sub quorate {
+ my ($self) = @_;
+
+ my ($node_info, $quorate) = $self->{hardware}->get_node_info();
+ my $node = $self->nodename();
+ return 0 if !$node_info->{$node}->{online};
+ return $quorate;
+}
+
+sub get_node_info {
+ my ($self) = @_;
+
+ return $self->{hardware}->get_node_info();
+}
+
+sub loop_start_hook {
+ my ($self, $starttime) = @_;
+
+ # do nothing, overwrite in subclass
+}
+
+sub loop_end_hook {
+ my ($self) = @_;
+
+ # do nothing, overwrite in subclass
+}
+
+sub watchdog_open {
+ my ($self) = @_;
+
+ my $node = $self->nodename();
+
+ return $self->{hardware}->watchdog_open($node);
+}
+
+sub watchdog_update {
+ my ($self, $wfh) = @_;
+
+ return $self->{hardware}->watchdog_update($wfh);
+}
+
+sub watchdog_close {
+ my ($self, $wfh) = @_;
+
+ return $self->{hardware}->watchdog_close($wfh);
+}
+
+sub exec_resource_agent {
+ my ($self, $sid, $cmd, @params) = @_;
+
+ die "implement me";
+}
+
+1;
--- /dev/null
+package PVE::HA::Sim::Hardware;
+
+# Simulate Hardware resources
+
+# power supply for nodes: on/off
+# network connection to nodes: on/off
+# watchdog devices for nodes
+
+use strict;
+use warnings;
+use POSIX qw(strftime EINTR);
+use Data::Dumper;
+use JSON;
+use IO::File;
+use Fcntl qw(:DEFAULT :flock);
+use File::Copy;
+use File::Path qw(make_path remove_tree);
+use PVE::HA::Groups;
+
+PVE::HA::Groups->register();
+PVE::HA::Groups->init();
+
+my $watchdog_timeout = 60;
+
+
+# Status directory layout
+#
+# configuration
+#
+# $testdir/cmdlist Command list for simulation
+# $testdir/hardware_status Hardware description (number of nodes, ...)
+# $testdir/manager_status CRM status (start with {})
+# $testdir/service_config Service configuration
+# $testdir/groups HA groups configuration
+# $testdir/service_status_<node> Service status
+
+#
+# runtime status for simulation system
+#
+# $testdir/status/cluster_locks Cluster locks
+# $testdir/status/hardware_status Hardware status (power/network on/off)
+# $testdir/status/watchdog_status Watchdog status
+#
+# runtime status
+#
+# $testdir/status/lrm_status_<node> LRM status
+# $testdir/status/manager_status CRM status
+# $testdir/status/crm_commands CRM command queue
+# $testdir/status/service_config Service configuration
+# $testdir/status/service_status_<node> Service status
+# $testdir/status/groups HA groups configuration
+
+sub read_lrm_status {
+ my ($self, $node) = @_;
+
+ my $filename = "$self->{statusdir}/lrm_status_$node";
+
+ return PVE::HA::Tools::read_json_from_file($filename, {});
+}
+
+sub write_lrm_status {
+ my ($self, $node, $status_obj) = @_;
+
+ my $filename = "$self->{statusdir}/lrm_status_$node";
+
+ PVE::HA::Tools::write_json_to_file($filename, $status_obj);
+}
+
+sub read_hardware_status_nolock {
+ my ($self) = @_;
+
+ my $filename = "$self->{statusdir}/hardware_status";
+
+ my $raw = PVE::Tools::file_get_contents($filename);
+ my $cstatus = decode_json($raw);
+
+ return $cstatus;
+}
+
+sub write_hardware_status_nolock {
+ my ($self, $cstatus) = @_;
+
+ my $filename = "$self->{statusdir}/hardware_status";
+
+ PVE::Tools::file_set_contents($filename, encode_json($cstatus));
+};
+
+sub read_service_config {
+ my ($self) = @_;
+
+ my $filename = "$self->{statusdir}/service_config";
+ my $conf = PVE::HA::Tools::read_json_from_file($filename);
+
+ foreach my $sid (keys %$conf) {
+ my $d = $conf->{$sid};
+
+ die "service '$sid' without assigned node!" if !$d->{node};
+
+ if ($sid =~ m/^pvevm:(\d+)$/) {
+ $d->{type} = 'pvevm';
+ $d->{name} = $1;
+ } else {
+ die "implement me";
+ }
+ $d->{state} = 'disabled' if !$d->{state};
+ }
+
+ return $conf;
+}
+
+sub write_service_config {
+ my ($self, $conf) = @_;
+
+ $self->{service_config} = $conf;
+
+ my $filename = "$self->{statusdir}/service_config";
+ return PVE::HA::Tools::write_json_to_file($filename, $conf);
+}
+
+sub change_service_location {
+ my ($self, $sid, $node) = @_;
+
+ my $conf = $self->read_service_config();
+
+ die "no such service '$sid'\n" if !$conf->{$sid};
+
+ $conf->{$sid}->{node} = $node;
+
+ $self->write_service_config($conf);
+}
+
+sub queue_crm_commands {
+ my ($self, $cmd) = @_;
+
+ chomp $cmd;
+
+ my $code = sub {
+ my $data = '';
+ my $filename = "$self->{statusdir}/crm_commands";
+ if (-f $filename) {
+ $data = PVE::Tools::file_get_contents($filename);
+ }
+ $data .= "$cmd\n";
+ PVE::Tools::file_set_contents($filename, $data);
+ };
+
+ $self->global_lock($code);
+
+ return undef;
+}
+
+sub read_crm_commands {
+ my ($self) = @_;
+
+ my $code = sub {
+ my $data = '';
+
+ my $filename = "$self->{statusdir}/crm_commands";
+ if (-f $filename) {
+ $data = PVE::Tools::file_get_contents($filename);
+ }
+ PVE::Tools::file_set_contents($filename, '');
+
+ return $data;
+ };
+
+ return $self->global_lock($code);
+}
+
+sub read_group_config {
+ my ($self) = @_;
+
+ my $filename = "$self->{statusdir}/groups";
+ my $raw = '';
+ $raw = PVE::Tools::file_get_contents($filename) if -f $filename;
+
+ return PVE::HA::Groups->parse_config($filename, $raw);
+}
+
+sub read_service_status {
+ my ($self, $node) = @_;
+
+ my $filename = "$self->{statusdir}/service_status_$node";
+ return PVE::HA::Tools::read_json_from_file($filename);
+}
+
+sub write_service_status {
+ my ($self, $node, $data) = @_;
+
+ my $filename = "$self->{statusdir}/service_status_$node";
+ my $res = PVE::HA::Tools::write_json_to_file($filename, $data);
+
+ # fixme: add test if a service runs on two nodes!!!
+
+ return $res;
+}
+
+my $default_group_config = <<__EOD;
+group: prefer_node1
+ nodes node1
+
+group: prefer_node2
+ nodes node2
+
+group: prefer_node3
+ nodes node3
+__EOD
+
+sub new {
+ my ($this, $testdir) = @_;
+
+ die "missing testdir" if !$testdir;
+
+ my $class = ref($this) || $this;
+
+ my $self = bless {}, $class;
+
+ my $statusdir = $self->{statusdir} = "$testdir/status";
+
+ remove_tree($statusdir);
+ mkdir $statusdir;
+
+ # copy initial configuartion
+ copy("$testdir/manager_status", "$statusdir/manager_status"); # optional
+
+ if (-f "$testdir/groups") {
+ copy("$testdir/groups", "$statusdir/groups");
+ } else {
+ PVE::Tools::file_set_contents("$statusdir/groups", $default_group_config);
+ }
+
+ if (-f "$testdir/service_config") {
+ copy("$testdir/service_config", "$statusdir/service_config");
+ } else {
+ my $conf = {
+ 'pvevm:101' => { node => 'node1', group => 'prefer_node1' },
+ 'pvevm:102' => { node => 'node2', group => 'prefer_node2' },
+ 'pvevm:103' => { node => 'node3', group => 'prefer_node3' },
+ 'pvevm:104' => { node => 'node1', group => 'prefer_node1' },
+ 'pvevm:105' => { node => 'node2', group => 'prefer_node2' },
+ 'pvevm:106' => { node => 'node3', group => 'prefer_node3' },
+ };
+ $self->write_service_config($conf);
+ }
+
+ if (-f "$testdir/hardware_status") {
+ copy("$testdir/hardware_status", "$statusdir/hardware_status") ||
+ die "Copy failed: $!\n";
+ } else {
+ my $cstatus = {
+ node1 => { power => 'off', network => 'off' },
+ node2 => { power => 'off', network => 'off' },
+ node3 => { power => 'off', network => 'off' },
+ };
+ $self->write_hardware_status_nolock($cstatus);
+ }
+
+
+ my $cstatus = $self->read_hardware_status_nolock();
+
+ foreach my $node (sort keys %$cstatus) {
+ $self->{nodes}->{$node} = {};
+
+ if (-f "$testdir/service_status_$node") {
+ copy("$testdir/service_status_$node", "$statusdir/service_status_$node");
+ } else {
+ $self->write_service_status($node, {});
+ }
+ }
+
+ $self->{service_config} = $self->read_service_config();
+
+ return $self;
+}
+
+sub get_time {
+ my ($self) = @_;
+
+ die "implement in subclass";
+}
+
+sub log {
+ my ($self, $level, $msg, $id) = @_;
+
+ chomp $msg;
+
+ my $time = $self->get_time();
+
+ $id = 'hardware' if !$id;
+
+ printf("%-5s %5d %12s: $msg\n", $level, $time, $id);
+}
+
+sub statusdir {
+ my ($self, $node) = @_;
+
+ return $self->{statusdir};
+}
+
+sub global_lock {
+ my ($self, $code, @param) = @_;
+
+ my $lockfile = "$self->{statusdir}/hardware.lck";
+ my $fh = IO::File->new(">>$lockfile") ||
+ die "unable to open '$lockfile'\n";
+
+ my $success;
+ for (;;) {
+ $success = flock($fh, LOCK_EX);
+ if ($success || ($! != EINTR)) {
+ last;
+ }
+ if (!$success) {
+ close($fh);
+ die "can't aquire lock '$lockfile' - $!\n";
+ }
+ }
+
+ my $res;
+
+ eval { $res = &$code($fh, @param) };
+ my $err = $@;
+
+ close($fh);
+
+ die $err if $err;
+
+ return $res;
+}
+
+my $compute_node_info = sub {
+ my ($self, $cstatus) = @_;
+
+ my $node_info = {};
+
+ my $node_count = 0;
+ my $online_count = 0;
+
+ foreach my $node (keys %$cstatus) {
+ my $d = $cstatus->{$node};
+
+ my $online = ($d->{power} eq 'on' && $d->{network} eq 'on') ? 1 : 0;
+ $node_info->{$node}->{online} = $online;
+
+ $node_count++;
+ $online_count++ if $online;
+ }
+
+ my $quorate = ($online_count > int($node_count/2)) ? 1 : 0;
+
+ if (!$quorate) {
+ foreach my $node (keys %$cstatus) {
+ my $d = $cstatus->{$node};
+ $node_info->{$node}->{online} = 0;
+ }
+ }
+
+ return ($node_info, $quorate);
+};
+
+sub get_node_info {
+ my ($self) = @_;
+
+ my ($node_info, $quorate);
+
+ my $code = sub {
+ my $cstatus = $self->read_hardware_status_nolock();
+ ($node_info, $quorate) = &$compute_node_info($self, $cstatus);
+ };
+
+ $self->global_lock($code);
+
+ return ($node_info, $quorate);
+}
+
+# simulate hardware commands
+# power <node> <on|off>
+# network <node> <on|off>
+
+sub sim_hardware_cmd {
+ my ($self, $cmdstr, $logid) = @_;
+
+ die "implement in subclass";
+}
+
+sub run {
+ my ($self) = @_;
+
+ die "implement in subclass";
+}
+
+my $modify_watchog = sub {
+ my ($self, $code) = @_;
+
+ my $update_cmd = sub {
+
+ my $filename = "$self->{statusdir}/watchdog_status";
+
+ my ($res, $wdstatus);
+
+ if (-f $filename) {
+ my $raw = PVE::Tools::file_get_contents($filename);
+ $wdstatus = decode_json($raw);
+ } else {
+ $wdstatus = {};
+ }
+
+ ($wdstatus, $res) = &$code($wdstatus);
+
+ PVE::Tools::file_set_contents($filename, encode_json($wdstatus));
+
+ return $res;
+ };
+
+ return $self->global_lock($update_cmd);
+};
+
+sub watchdog_check {
+ my ($self, $node) = @_;
+
+ my $code = sub {
+ my ($wdstatus) = @_;
+
+ my $res = 1;
+
+ foreach my $wfh (keys %$wdstatus) {
+ my $wd = $wdstatus->{$wfh};
+ next if $wd->{node} ne $node;
+
+ my $ctime = $self->get_time();
+ my $tdiff = $ctime - $wd->{update_time};
+
+ if ($tdiff > $watchdog_timeout) { # expired
+ $res = 0;
+ delete $wdstatus->{$wfh};
+ }
+ }
+
+ return ($wdstatus, $res);
+ };
+
+ return &$modify_watchog($self, $code);
+}
+
+my $wdcounter = 0;
+
+sub watchdog_open {
+ my ($self, $node) = @_;
+
+ my $code = sub {
+ my ($wdstatus) = @_;
+
+ ++$wdcounter;
+
+ my $id = "WD:$node:$$:$wdcounter";
+
+ die "internal error" if defined($wdstatus->{$id});
+
+ $wdstatus->{$id} = {
+ node => $node,
+ update_time => $self->get_time(),
+ };
+
+ return ($wdstatus, $id);
+ };
+
+ return &$modify_watchog($self, $code);
+}
+
+sub watchdog_close {
+ my ($self, $wfh) = @_;
+
+ my $code = sub {
+ my ($wdstatus) = @_;
+
+ my $wd = $wdstatus->{$wfh};
+ die "no such watchdog handle '$wfh'\n" if !defined($wd);
+
+ my $tdiff = $self->get_time() - $wd->{update_time};
+ die "watchdog expired" if $tdiff > $watchdog_timeout;
+
+ delete $wdstatus->{$wfh};
+
+ return ($wdstatus);
+ };
+
+ return &$modify_watchog($self, $code);
+}
+
+sub watchdog_update {
+ my ($self, $wfh) = @_;
+
+ my $code = sub {
+ my ($wdstatus) = @_;
+
+ my $wd = $wdstatus->{$wfh};
+
+ die "no such watchdog handle '$wfh'\n" if !defined($wd);
+
+ my $ctime = $self->get_time();
+ my $tdiff = $ctime - $wd->{update_time};
+
+ die "watchdog expired" if $tdiff > $watchdog_timeout;
+
+ $wd->{update_time} = $ctime;
+
+ return ($wdstatus);
+ };
+
+ return &$modify_watchog($self, $code);
+}
+
+
+
+1;
--- /dev/null
+SOURCES=Env.pm Hardware.pm TestEnv.pm TestHardware.pm RTEnv.pm RTHardware.pm
+
+.PHONY: installsim
+installsim:
+ install -d -m 0755 ${DESTDIR}${PERLDIR}/PVE/HA/Sim
+ for i in ${SOURCES}; do install -D -m 0644 $$i ${DESTDIR}${PERLDIR}/PVE/HA/Sim/$$i; done
--- /dev/null
+package PVE::HA::Sim::RTEnv;
+
+use strict;
+use warnings;
+use POSIX qw(strftime EINTR);
+use Data::Dumper;
+use JSON;
+use IO::File;
+use Fcntl qw(:DEFAULT :flock);
+
+use PVE::HA::Tools;
+
+use base qw(PVE::HA::Sim::Env);
+
+sub new {
+ my ($this, $nodename, $hardware, $log_id) = @_;
+
+ my $class = ref($this) || $this;
+
+ my $self = $class->SUPER::new($nodename, $hardware, $log_id);
+
+ return $self;
+}
+
+sub get_time {
+ my ($self) = @_;
+
+ return time();
+}
+
+sub log {
+ my ($self, $level, $msg) = @_;
+
+ chomp $msg;
+
+ my $time = $self->get_time();
+
+ printf("%-5s %10s %12s: $msg\n", $level, strftime("%H:%M:%S", localtime($time)),
+ "$self->{nodename}/$self->{log_id}");
+}
+
+sub sleep {
+ my ($self, $delay) = @_;
+
+ CORE::sleep($delay);
+}
+
+sub sleep_until {
+ my ($self, $end_time) = @_;
+
+ for (;;) {
+ my $cur_time = time();
+
+ last if $cur_time >= $end_time;
+
+ $self->sleep(1);
+ }
+}
+
+sub loop_start_hook {
+ my ($self) = @_;
+
+ $self->{loop_start} = $self->get_time();
+}
+
+sub loop_end_hook {
+ my ($self) = @_;
+
+ my $delay = $self->get_time() - $self->{loop_start};
+
+ die "loop take too long ($delay seconds)\n" if $delay > 30;
+}
+
+sub exec_resource_agent {
+ my ($self, $sid, $cmd, @params) = @_;
+
+ my $hardware = $self->{hardware};
+
+ my $nodename = $self->{nodename};
+
+ my $sc = $hardware->read_service_config($nodename);
+
+ # fixme: return valid_exit code (instead of using die)
+ my $cd = $sc->{$sid};
+ die "no such service" if !$cd;
+
+ my $ss = $hardware->read_service_status($nodename);
+
+ if ($cmd eq 'started') {
+
+ # fixme: return valid_exit code
+ die "service '$sid' not on this node" if $cd->{node} ne $nodename;
+
+ if ($ss->{$sid}) {
+ $self->log("info", "service status $sid: running");
+ return 0;
+ }
+ $self->log("info", "starting service $sid");
+
+ $self->sleep(2);
+
+ $ss->{$sid} = 1;
+ $hardware->write_service_status($nodename, $ss);
+
+ $self->log("info", "service status $sid started");
+
+ return 0;
+
+ } elsif ($cmd eq 'request_stop' || $cmd eq 'stopped') {
+
+ # fixme: return valid_exit code
+ die "service '$sid' not on this node" if $cd->{node} ne $nodename;
+
+ if (!$ss->{$sid}) {
+ $self->log("info", "service status $sid: stopped");
+ return 0;
+ }
+ $self->log("info", "stopping service $sid");
+
+ $self->sleep(2);
+
+ $ss->{$sid} = 0;
+ $hardware->write_service_status($nodename, $ss);
+
+ $self->log("info", "service status $sid stopped");
+
+ return 0;
+
+ } elsif ($cmd eq 'migrate' || $cmd eq 'relocate') {
+
+ my $target = $params[0];
+ die "$cmd '$sid' failed - missing target\n" if !defined($target);
+
+ if ($cd->{node} eq $target) {
+ # already migrate
+ return 0;
+ } elsif ($cd->{node} eq $nodename) {
+
+ $self->log("info", "service $sid - start $cmd to node '$target'");
+
+ if ($cmd eq 'relocate' && $ss->{$sid}) {
+ $self->log("info", "stopping service $sid (relocate)");
+ $self->sleep(1);
+ $ss->{$sid} = 0;
+ $hardware->write_service_status($nodename, $ss);
+ $self->log("info", "service status $sid stopped");
+ }
+
+ $self->sleep(2);
+ $self->change_service_location($sid, $target);
+ $self->log("info", "service $sid - end $cmd to node '$target'");
+
+ return 0;
+
+ } else {
+ die "migrate '$sid' failed - service is not on this node\n";
+ }
+
+
+ }
+
+ die "implement me (cmd '$cmd')";
+}
+
+1;
--- /dev/null
+package PVE::HA::Sim::RTHardware;
+
+# Simulate Hardware resources in Realtime by
+# running CRM and LRM in separate processes
+
+use strict;
+use warnings;
+use POSIX qw(strftime EINTR);
+use Data::Dumper;
+use JSON;
+use IO::File;
+use IO::Select;
+use Fcntl qw(:DEFAULT :flock);
+use File::Copy;
+use File::Path qw(make_path remove_tree);
+
+use Glib;
+
+use Gtk3 '-init';
+
+use PVE::HA::CRM;
+use PVE::HA::LRM;
+
+use PVE::HA::Sim::RTEnv;
+use base qw(PVE::HA::Sim::Hardware);
+
+sub new {
+ my ($this, $testdir) = @_;
+
+ my $class = ref($this) || $this;
+
+ my $self = $class->SUPER::new($testdir);
+
+ foreach my $node (sort keys %{$self->{nodes}}) {
+ my $d = $self->{nodes}->{$node};
+
+ $d->{crm} = undef; # create on power on
+ $d->{lrm} = undef; # create on power on
+ }
+
+ $self->create_main_window();
+
+ return $self;
+}
+
+sub get_time {
+ my ($self) = @_;
+
+ return time();
+}
+
+sub log {
+ my ($self, $level, $msg, $id) = @_;
+
+ chomp $msg;
+
+ my $time = $self->get_time();
+
+ $id = 'hardware' if !$id;
+
+ my $text = sprintf("%-5s %10s %12s: $msg\n", $level,
+ strftime("%H:%M:%S", localtime($time)), $id);
+
+ $self->append_text($text);
+}
+
+# fixme: duplicate code in Env?
+sub read_manager_status {
+ my ($self) = @_;
+
+ my $filename = "$self->{statusdir}/manager_status";
+
+ return PVE::HA::Tools::read_json_from_file($filename, {});
+}
+
+sub fork_daemon {
+ my ($self, $lockfh, $type, $node) = @_;
+
+ my @psync = POSIX::pipe();
+
+ my $pid = fork();
+ die "fork failed" if ! defined($pid);
+
+ if ($pid == 0) {
+
+ close($lockfh) if defined($lockfh); # unlock global lock
+
+ POSIX::close($psync[0]);
+
+ my $outfh = $psync[1];
+
+ my $fd = fileno (STDIN);
+ close STDIN;
+ POSIX::close(0) if $fd != 0;
+
+ die "unable to redirect STDIN - $!"
+ if !open(STDIN, "</dev/null");
+
+ # redirect STDOUT
+ $fd = fileno(STDOUT);
+ close STDOUT;
+ POSIX::close (1) if $fd != 1;
+
+ die "unable to redirect STDOUT - $!"
+ if !open(STDOUT, ">&", $outfh);
+
+ STDOUT->autoflush (1);
+
+ # redirect STDERR to STDOUT
+ $fd = fileno(STDERR);
+ close STDERR;
+ POSIX::close(2) if $fd != 2;
+
+ die "unable to redirect STDERR - $!"
+ if !open(STDERR, ">&1");
+
+ STDERR->autoflush(1);
+
+ if ($type eq 'crm') {
+
+ my $haenv = PVE::HA::Env->new('PVE::HA::Sim::RTEnv', $node, $self, 'crm');
+
+ my $crm = PVE::HA::CRM->new($haenv);
+
+ for (;;) {
+ $haenv->loop_start_hook();
+
+ if (!$crm->do_one_iteration()) {
+ $haenv->log("info", "daemon stopped");
+ exit (0);
+ }
+
+ $haenv->loop_end_hook();
+ }
+
+ } else {
+
+ my $haenv = PVE::HA::Env->new('PVE::HA::Sim::RTEnv', $node, $self, 'lrm');
+
+ my $lrm = PVE::HA::LRM->new($haenv);
+
+ for (;;) {
+ $haenv->loop_start_hook();
+
+ if (!$lrm->do_one_iteration()) {
+ $haenv->log("info", "daemon stopped");
+ exit (0);
+ }
+
+ $haenv->loop_end_hook();
+ }
+ }
+
+ exit(-1);
+ }
+
+ # parent
+
+ POSIX::close ($psync[1]);
+
+ Glib::IO->add_watch($psync[0], ['in', 'hup'], sub {
+ my ($fd, $cond) = @_;
+ if ($cond eq 'in') {
+ my $readbuf;
+ if (my $count = POSIX::read($fd, $readbuf, 8192)) {
+ $self->append_text($readbuf);
+ }
+ return 1;
+ } else {
+ POSIX::close($fd);
+ return 0;
+ }
+ });
+
+ return $pid;
+}
+
+# simulate hardware commands
+# power <node> <on|off>
+# network <node> <on|off>
+
+sub sim_hardware_cmd {
+ my ($self, $cmdstr, $logid) = @_;
+
+ my $cstatus;
+
+ # note: do not fork when we own the lock!
+ my $code = sub {
+ my ($lockfh) = @_;
+
+ $cstatus = $self->read_hardware_status_nolock();
+
+ my ($cmd, $node, $action) = split(/\s+/, $cmdstr);
+
+ die "sim_hardware_cmd: no node specified" if !$node;
+ die "sim_hardware_cmd: unknown action '$action'" if $action !~ m/^(on|off)$/;
+
+ my $d = $self->{nodes}->{$node};
+ die "sim_hardware_cmd: no such node '$node'\n" if !$d;
+
+ $self->log('info', "execute $cmdstr", $logid);
+
+ if ($cmd eq 'power') {
+ if ($cstatus->{$node}->{power} ne $action) {
+ if ($action eq 'on') {
+ $d->{crm} = $self->fork_daemon($lockfh, 'crm', $node) if !$d->{crm};
+ $d->{lrm} = $self->fork_daemon($lockfh, 'lrm', $node) if !$d->{lrm};
+ } else {
+ if ($d->{crm}) {
+ $self->log('info', "crm on node '$node' killed by poweroff");
+ kill(9, $d->{crm});
+ $d->{crm} = undef;
+ }
+ if ($d->{lrm}) {
+ $self->log('info', "lrm on node '$node' killed by poweroff");
+ kill(9, $d->{lrm});
+ $d->{lrm} = undef;
+ }
+ }
+ }
+
+ $cstatus->{$node}->{power} = $action;
+ $cstatus->{$node}->{network} = $action;
+
+ } elsif ($cmd eq 'network') {
+ $cstatus->{$node}->{network} = $action;
+ } else {
+ die "sim_hardware_cmd: unknown command '$cmd'\n";
+ }
+
+ $self->write_hardware_status_nolock($cstatus);
+ };
+
+ my $res = $self->global_lock($code);
+
+ # update GUI outside lock
+
+ foreach my $node (keys %$cstatus) {
+ my $d = $self->{nodes}->{$node};
+ $d->{network_btn}->set_active($cstatus->{$node}->{network} eq 'on');
+ $d->{power_btn}->set_active($cstatus->{$node}->{power} eq 'on');
+ }
+
+ return $res;
+}
+
+sub cleanup {
+ my ($self) = @_;
+
+ my @nodes = sort keys %{$self->{nodes}};
+ foreach my $node (@nodes) {
+ my $d = $self->{nodes}->{$node};
+
+ if ($d->{crm}) {
+ kill 9, $d->{crm};
+ delete $d->{crm};
+ }
+ if ($d->{lrm}) {
+ kill 9, $d->{lrm};
+ delete $d->{lrm};
+ }
+ }
+}
+
+sub append_text {
+ my ($self, $text) = @_;
+
+ my $logview = $self->{gui}->{text_view} || die "GUI not ready";
+ my $textbuf = $logview->get_buffer();
+
+ $textbuf->insert_at_cursor($text, -1);
+ my $lines = $textbuf->get_line_count();
+
+ my $history = 102;
+
+ if ($lines > $history) {
+ my $start = $textbuf->get_iter_at_line(0);
+ my $end = $textbuf->get_iter_at_line($lines - $history);
+ $textbuf->delete($start, $end);
+ }
+
+ $logview->scroll_to_mark($textbuf->get_insert(), 0.0, 1, 0.0, 1.0);
+}
+
+sub set_power_state {
+ my ($self, $node) = @_;
+
+ my $d = $self->{nodes}->{$node} || die "no such node '$node'";
+
+ my $action = $d->{power_btn}->get_active() ? 'on' : 'off';
+
+ $self->sim_hardware_cmd("power $node $action");
+}
+
+sub set_network_state {
+ my ($self, $node) = @_;
+
+ my $d = $self->{nodes}->{$node} || die "no such node '$node'";
+
+ my $action = $d->{network_btn}->get_active() ? 'on' : 'off';
+
+ $self->sim_hardware_cmd("network $node $action");
+}
+
+sub set_service_state {
+ my ($self, $sid) = @_;
+
+ $self->{service_config} = $self->read_service_config();
+
+ my $d = $self->{service_gui}->{$sid} || die "no such service '$sid'";
+
+ my $state = $d->{enable_btn}->get_active() ? 'enabled' : 'disabled';
+
+ $d = $self->{service_config}->{$sid} || die "no such service '$sid'";
+
+ $d->{state} = $state;
+
+ $self->write_service_config($self->{service_config});
+}
+
+sub create_node_control {
+ my ($self) = @_;
+
+ my $ngrid = Gtk3::Grid->new();
+ $ngrid->set_row_spacing(2);
+ $ngrid->set_column_spacing(5);
+ $ngrid->set('margin-left', 5);
+
+ my $w = Gtk3::Label->new('Node');
+ $ngrid->attach($w, 0, 0, 1, 1);
+ $w = Gtk3::Label->new('Power');
+ $ngrid->attach($w, 1, 0, 1, 1);
+ $w = Gtk3::Label->new('Network');
+ $ngrid->attach($w, 2, 0, 1, 1);
+ $w = Gtk3::Label->new('Status');
+ $w->set_size_request(150, -1);
+ $w->set_alignment (0, 0.5);
+ $ngrid->attach($w, 3, 0, 1, 1);
+
+ my $row = 1;
+
+ my @nodes = sort keys %{$self->{nodes}};
+
+ foreach my $node (@nodes) {
+ my $d = $self->{nodes}->{$node};
+
+ $w = Gtk3::Label->new($node);
+ $ngrid->attach($w, 0, $row, 1, 1);
+ $w = Gtk3::Switch->new();
+ $ngrid->attach($w, 1, $row, 1, 1);
+ $d->{power_btn} = $w;
+ $w->signal_connect('notify::active' => sub {
+ $self->set_power_state($node);
+ }),
+
+ $w = Gtk3::Switch->new();
+ $ngrid->attach($w, 2, $row, 1, 1);
+ $d->{network_btn} = $w;
+ $w->signal_connect('notify::active' => sub {
+ $self->set_network_state($node);
+ }),
+
+ $w = Gtk3::Label->new('-');
+ $w->set_alignment (0, 0.5);
+ $ngrid->attach($w, 3, $row, 1, 1);
+ $d->{node_status_label} = $w;
+
+ $row++;
+ }
+
+ return $ngrid;
+}
+
+sub show_migrate_dialog {
+ my ($self, $sid) = @_;
+
+ my $dialog = Gtk3::Dialog->new();
+
+ $dialog->set_title("Migrate $sid");
+ $dialog->set_modal(1);
+
+ my $grid = Gtk3::Grid->new();
+ $grid->set_row_spacing(2);
+ $grid->set_column_spacing(5);
+ $grid->set('margin', 5);
+
+ my $w = Gtk3::Label->new('Target Mode');
+ $grid->attach($w, 0, 0, 1, 1);
+
+ my @nodes = sort keys %{$self->{nodes}};
+ $w = Gtk3::ComboBoxText->new();
+ foreach my $node (@nodes) {
+ $w->append_text($node);
+ }
+
+ my $target = '';
+ $w->signal_connect('notify::active' => sub {
+ my $w = shift;
+
+ my $sel = $w->get_active();
+ return if $sel < 0;
+
+ $target = $nodes[$sel];
+ });
+ $grid->attach($w, 1, 0, 1, 1);
+
+ my $relocate_btn = Gtk3::CheckButton->new_with_label("stop service (relocate)");
+ $grid->attach($relocate_btn, 1, 1, 1, 1);
+
+ my $contarea = $dialog->get_content_area();
+
+ $contarea->add($grid);
+
+ $dialog->add_button("_OK", 1);
+
+ $dialog->show_all();
+ my $res = $dialog->run();
+
+ $dialog->destroy();
+
+ if ($res == 1 && $target) {
+ if ($relocate_btn->get_active()) {
+ $self->queue_crm_commands("relocate $sid $target");
+ } else {
+ $self->queue_crm_commands("migrate $sid $target");
+ }
+ }
+}
+
+sub create_service_control {
+ my ($self) = @_;
+
+ my $sgrid = Gtk3::Grid->new();
+ $sgrid->set_row_spacing(2);
+ $sgrid->set_column_spacing(5);
+ $sgrid->set('margin', 5);
+
+ my $w = Gtk3::Label->new('Service');
+ $sgrid->attach($w, 0, 0, 1, 1);
+ $w = Gtk3::Label->new('Enable');
+ $sgrid->attach($w, 1, 0, 1, 1);
+ $w = Gtk3::Label->new('Node');
+ $sgrid->attach($w, 3, 0, 1, 1);
+ $w = Gtk3::Label->new('Status');
+ $w->set_alignment (0, 0.5);
+ $w->set_size_request(150, -1);
+ $sgrid->attach($w, 4, 0, 1, 1);
+
+ my $row = 1;
+ my @nodes = keys %{$self->{nodes}};
+
+ foreach my $sid (sort keys %{$self->{service_config}}) {
+ my $d = $self->{service_config}->{$sid};
+
+ $w = Gtk3::Label->new($sid);
+ $sgrid->attach($w, 0, $row, 1, 1);
+
+ $w = Gtk3::Switch->new();
+ $sgrid->attach($w, 1, $row, 1, 1);
+ $w->set_active(1) if $d->{state} eq 'enabled';
+ $self->{service_gui}->{$sid}->{enable_btn} = $w;
+ $w->signal_connect('notify::active' => sub {
+ $self->set_service_state($sid);
+ }),
+
+
+ $w = Gtk3::Button->new('Migrate');
+ $sgrid->attach($w, 2, $row, 1, 1);
+ $w->signal_connect(clicked => sub {
+ $self->show_migrate_dialog($sid);
+ });
+
+ $w = Gtk3::Label->new($d->{node});
+ $sgrid->attach($w, 3, $row, 1, 1);
+ $self->{service_gui}->{$sid}->{node_label} = $w;
+
+ $w = Gtk3::Label->new('-');
+ $w->set_alignment (0, 0.5);
+ $sgrid->attach($w, 4, $row, 1, 1);
+ $self->{service_gui}->{$sid}->{status_label} = $w;
+
+ $row++;
+ }
+
+ return $sgrid;
+}
+
+sub create_log_view {
+ my ($self) = @_;
+
+ my $nb = Gtk3::Notebook->new();
+
+ my $l1 = Gtk3::Label->new('Cluster Log');
+
+ my $logview = Gtk3::TextView->new();
+ $logview->set_editable(0);
+ $logview->set_cursor_visible(0);
+
+ $self->{gui}->{text_view} = $logview;
+
+ my $swindow = Gtk3::ScrolledWindow->new();
+ $swindow->set_size_request(1024, 768);
+ $swindow->add($logview);
+
+ $nb->insert_page($swindow, $l1, 0);
+
+ my $l2 = Gtk3::Label->new('Manager Status');
+
+ my $statview = Gtk3::TextView->new();
+ $statview->set_editable(0);
+ $statview->set_cursor_visible(0);
+
+ $self->{gui}->{stat_view} = $statview;
+
+ $swindow = Gtk3::ScrolledWindow->new();
+ $swindow->set_size_request(640, 400);
+ $swindow->add($statview);
+
+ $nb->insert_page($swindow, $l2, 1);
+ return $nb;
+}
+
+sub create_main_window {
+ my ($self) = @_;
+
+ my $window = Gtk3::Window->new();
+ $window->set_title("Proxmox HA Simulator");
+
+ $window->signal_connect( destroy => sub { Gtk3::main_quit(); });
+
+ my $grid = Gtk3::Grid->new();
+
+ my $frame = $self->create_log_view();
+ $grid->attach($frame, 0, 0, 1, 1);
+ $frame->set('expand', 1);
+
+ my $vbox = Gtk3::VBox->new(0, 0);
+ $grid->attach($vbox, 1, 0, 1, 1);
+
+ my $ngrid = $self->create_node_control();
+ $vbox->pack_start($ngrid, 0, 0, 0);
+
+ my $sep = Gtk3::HSeparator->new;
+ $sep->set('margin-top', 10);
+ $vbox->pack_start ($sep, 0, 0, 0);
+
+ my $sgrid = $self->create_service_control();
+ $vbox->pack_start($sgrid, 0, 0, 0);
+
+ $window->add($grid);
+
+ $window->show_all;
+ $window->realize ();
+}
+
+sub run {
+ my ($self) = @_;
+
+ Glib::Timeout->add(1000, sub {
+
+ $self->{service_config} = $self->read_service_config();
+
+ # check all watchdogs
+ my @nodes = sort keys %{$self->{nodes}};
+ foreach my $node (@nodes) {
+ if (!$self->watchdog_check($node)) {
+ $self->sim_hardware_cmd("power $node off", 'watchdog');
+ $self->log('info', "server '$node' stopped by poweroff (watchdog)");
+ }
+ }
+
+ my $mstatus = $self->read_manager_status();
+ my $node_status = $mstatus->{node_status} || {};
+
+ foreach my $node (@nodes) {
+ my $ns = $node_status->{$node} || '-';
+ my $d = $self->{nodes}->{$node};
+ next if !$d;
+ my $sl = $d->{node_status_label};
+ next if !$sl;
+
+ if ($mstatus->{master_node} && ($mstatus->{master_node} eq $node)) {
+ $sl->set_text(uc($ns));
+ } else {
+ $sl->set_text($ns);
+ }
+ }
+
+ my $service_status = $mstatus->{service_status} || {};
+ my @services = sort keys %{$self->{service_config}};
+
+ foreach my $sid (@services) {
+ my $sc = $self->{service_config}->{$sid};
+ my $ss = $service_status->{$sid};
+ my $sgui = $self->{service_gui}->{$sid};
+ next if !$sgui;
+ my $nl = $sgui->{node_label};
+ $nl->set_text($sc->{node});
+
+ my $sl = $sgui->{status_label};
+ next if !$sl;
+
+ my $text = ($ss && $ss->{state}) ? $ss->{state} : '-';
+ $sl->set_text($text);
+ }
+
+ if (my $sv = $self->{gui}->{stat_view}) {
+ my $text = Dumper($mstatus);
+ my $textbuf = $sv->get_buffer();
+ $textbuf->set_text($text, -1);
+ }
+
+ return 1; # repeat
+ });
+
+ Gtk3->main;
+
+ $self->cleanup();
+}
+
+1;
--- /dev/null
+package PVE::HA::Sim::TestEnv;
+
+use strict;
+use warnings;
+use POSIX qw(strftime EINTR);
+use Data::Dumper;
+use JSON;
+use IO::File;
+use Fcntl qw(:DEFAULT :flock);
+
+use PVE::HA::Tools;
+
+use base qw(PVE::HA::Sim::Env);
+
+sub new {
+ my ($this, $nodename, $hardware, $log_id) = @_;
+
+ my $class = ref($this) || $this;
+
+ my $self = $class->SUPER::new($nodename, $hardware, $log_id);
+
+ $self->{cur_time} = 0;
+ $self->{loop_delay} = 0;
+
+ return $self;
+}
+
+sub get_time {
+ my ($self) = @_;
+
+ return $self->{cur_time};
+}
+
+sub sleep {
+ my ($self, $delay) = @_;
+
+ $self->{loop_delay} += $delay;
+}
+
+sub sleep_until {
+ my ($self, $end_time) = @_;
+
+ my $cur_time = $self->{cur_time} + $self->{loop_delay};
+
+ return if $cur_time >= $end_time;
+
+ $self->{loop_delay} += $end_time - $cur_time;
+}
+
+sub get_ha_manager_lock {
+ my ($self) = @_;
+
+ my $res = $self->SUPER::get_ha_manager_lock();
+ ++$self->{loop_delay};
+ return $res;
+}
+
+sub get_ha_agent_lock {
+ my ($self) = @_;
+
+ my $res = $self->SUPER::get_ha_agent_lock();
+ ++$self->{loop_delay};
+
+ return $res;
+}
+
+sub test_ha_agent_lock {
+ my ($self, $node) = @_;
+
+ my $res = $self->SUPER::test_ha_agent_lock($node);
+ ++$self->{loop_delay};
+ return $res;
+}
+
+sub loop_start_hook {
+ my ($self, $starttime) = @_;
+
+ $self->{loop_delay} = 0;
+
+ die "no starttime" if !defined($starttime);
+ die "strange start time" if $starttime < $self->{cur_time};
+
+ $self->{cur_time} = $starttime;
+
+ # do nothing
+}
+
+sub loop_end_hook {
+ my ($self) = @_;
+
+ my $delay = $self->{loop_delay};
+ $self->{loop_delay} = 0;
+
+ die "loop take too long ($delay seconds)\n" if $delay > 30;
+
+ # $self->{cur_time} += $delay;
+
+ $self->{cur_time} += 1; # easier for simulation
+}
+
+1;
--- /dev/null
+package PVE::HA::Sim::TestHardware;
+
+# Simulate Hardware resources
+
+# power supply for nodes: on/off
+# network connection to nodes: on/off
+# watchdog devices for nodes
+
+use strict;
+use warnings;
+use POSIX qw(strftime EINTR);
+use Data::Dumper;
+use JSON;
+use IO::File;
+use Fcntl qw(:DEFAULT :flock);
+use File::Copy;
+use File::Path qw(make_path remove_tree);
+
+use PVE::HA::CRM;
+use PVE::HA::LRM;
+
+use PVE::HA::Sim::TestEnv;
+use base qw(PVE::HA::Sim::Hardware);
+
+my $max_sim_time = 10000;
+
+sub new {
+ my ($this, $testdir) = @_;
+
+ my $class = ref($this) || $this;
+
+ my $self = $class->SUPER::new($testdir);
+
+ my $raw = PVE::Tools::file_get_contents("$testdir/cmdlist");
+ $self->{cmdlist} = decode_json($raw);
+
+ $self->{loop_count} = 0;
+ $self->{cur_time} = 0;
+
+ foreach my $node (sort keys %{$self->{nodes}}) {
+
+ my $d = $self->{nodes}->{$node};
+
+ $d->{crm_env} =
+ PVE::HA::Env->new('PVE::HA::Sim::TestEnv', $node, $self, 'crm');
+
+ $d->{lrm_env} =
+ PVE::HA::Env->new('PVE::HA::Sim::TestEnv', $node, $self, 'lrm');
+
+ $d->{crm} = undef; # create on power on
+ $d->{lrm} = undef; # create on power on
+ }
+
+ return $self;
+}
+
+sub get_time {
+ my ($self) = @_;
+
+ return $self->{cur_time};
+}
+
+# simulate hardware commands
+# power <node> <on|off>
+# network <node> <on|off>
+
+sub sim_hardware_cmd {
+ my ($self, $cmdstr, $logid) = @_;
+
+ my $code = sub {
+
+ my $cstatus = $self->read_hardware_status_nolock();
+
+ my ($cmd, $node, $action) = split(/\s+/, $cmdstr);
+
+ die "sim_hardware_cmd: no node specified" if !$node;
+ die "sim_hardware_cmd: unknown action '$action'" if $action !~ m/^(on|off)$/;
+
+ my $d = $self->{nodes}->{$node};
+ die "sim_hardware_cmd: no such node '$node'\n" if !$d;
+
+ $self->log('info', "execute $cmdstr", $logid);
+
+ if ($cmd eq 'power') {
+ if ($cstatus->{$node}->{power} ne $action) {
+ if ($action eq 'on') {
+ $d->{crm} = PVE::HA::CRM->new($d->{crm_env}) if !$d->{crm};
+ $d->{lrm} = PVE::HA::LRM->new($d->{lrm_env}) if !$d->{lrm};
+ } else {
+ if ($d->{crm}) {
+ $d->{crm_env}->log('info', "killed by poweroff");
+ $d->{crm} = undef;
+ }
+ if ($d->{lrm}) {
+ $d->{lrm_env}->log('info', "killed by poweroff");
+ $d->{lrm} = undef;
+ }
+ }
+ }
+
+ $cstatus->{$node}->{power} = $action;
+ $cstatus->{$node}->{network} = $action;
+
+ } elsif ($cmd eq 'network') {
+ $cstatus->{$node}->{network} = $action;
+ } else {
+ die "sim_hardware_cmd: unknown command '$cmd'\n";
+ }
+
+ $self->write_hardware_status_nolock($cstatus);
+ };
+
+ return $self->global_lock($code);
+}
+
+sub run {
+ my ($self) = @_;
+
+ my $last_command_time = 0;
+
+ for (;;) {
+
+ my $starttime = $self->get_time();
+
+ my @nodes = sort keys %{$self->{nodes}};
+
+ my $nodecount = scalar(@nodes);
+
+ my $looptime = $nodecount*2;
+ $looptime = 20 if $looptime < 20;
+
+ die "unable to simulate so many nodes. You need to increate watchdog/lock timeouts.\n"
+ if $looptime >= 60;
+
+ foreach my $node (@nodes) {
+
+ my $d = $self->{nodes}->{$node};
+
+ if (my $crm = $d->{crm}) {
+
+ $d->{crm_env}->loop_start_hook($self->get_time());
+
+ die "implement me (CRM exit)" if !$crm->do_one_iteration();
+
+ $d->{crm_env}->loop_end_hook();
+
+ my $nodetime = $d->{crm_env}->get_time();
+ $self->{cur_time} = $nodetime if $nodetime > $self->{cur_time};
+ }
+
+ if (my $lrm = $d->{lrm}) {
+
+ $d->{lrm_env}->loop_start_hook($self->get_time());
+
+ die "implement me (LRM exit)" if !$lrm->do_one_iteration();
+
+ $d->{lrm_env}->loop_end_hook();
+
+ my $nodetime = $d->{lrm_env}->get_time();
+ $self->{cur_time} = $nodetime if $nodetime > $self->{cur_time};
+ }
+
+ foreach my $n (@nodes) {
+ if (!$self->watchdog_check($n)) {
+ $self->sim_hardware_cmd("power $n off", 'watchdog');
+ $self->log('info', "server '$n' stopped by poweroff (watchdog)");
+ $self->{nodes}->{$n}->{crm} = undef;
+ $self->{nodes}->{$n}->{lrm} = undef;
+ }
+ }
+ }
+
+
+ $self->{cur_time} = $starttime + $looptime
+ if ($self->{cur_time} - $starttime) < $looptime;
+
+ die "simulation end\n" if $self->{cur_time} > $max_sim_time;
+
+ # apply new comand after 5 loop iterations
+
+ if (($self->{loop_count} % 5) == 0) {
+ my $list = shift $self->{cmdlist};
+ if (!$list) {
+ # end sumulation (500 seconds after last command)
+ return if (($self->{cur_time} - $last_command_time) > 500);
+ }
+
+ foreach my $cmd (@$list) {
+ $last_command_time = $self->{cur_time};
+ $self->sim_hardware_cmd($cmd, 'cmdlist');
+ }
+ }
+
+ ++$self->{loop_count};
+ }
+}
+
+1;
--- /dev/null
+package PVE::HA::Tools;
+
+use strict;
+use warnings;
+use JSON;
+use PVE::Tools;
+
+sub read_json_from_file {
+ my ($filename, $default) = @_;
+
+ my $data;
+
+ if (defined($default) && (! -f $filename)) {
+ $data = $default;
+ } else {
+ my $raw = PVE::Tools::file_get_contents($filename);
+ $data = decode_json($raw);
+ }
+
+ return $data;
+}
+
+sub write_json_to_file {
+ my ($filename, $data) = @_;
+
+ my $raw = encode_json($data);
+
+ PVE::Tools::file_set_contents($filename, $raw);
+}
+
+
+1;
--- /dev/null
+
+.PHONY: install
+install:
+ install -d -m 0755 ${DESTDIR}${PERLDIR}/PVE
+ make -C HA install
+
+.PHONY: installsim
+installsim:
+ install -d -m 0755 ${DESTDIR}${PERLDIR}/PVE
+ make -C HA installsim
--- /dev/null
+#!/usr/bin/perl
+
+use strict;
+use warnings;
+use PVE::SafeSyslog;
+use PVE::Daemon;
+use Data::Dumper;
+use PVE::RPCEnvironment;
+
+use PVE::HA::Env;
+use PVE::HA::Env::PVE2;
+use PVE::HA::CRM;
+
+use base qw(PVE::Daemon);
+
+my $cmdline = [$0, @ARGV];
+
+my %daemon_options = (stop_wait_time => 5);
+
+my $daemon = __PACKAGE__->new('pve-ha-crm', $cmdline, %daemon_options);
+
+my $rpcenv = PVE::RPCEnvironment->init('cli');
+
+$rpcenv->init_request();
+$rpcenv->set_language($ENV{LANG});
+$rpcenv->set_user('root@pam');
+
+sub run {
+ my ($self) = @_;
+
+ $self->{haenv} = PVE::HA::Env->new('PVE::HA::Env::PVE2', $self->{nodename});
+
+ $self->{crm} = PVE::HA::CRM->new($self->{haenv});
+
+ for (;;) {
+ $self->{haenv}->loop_start_hook();
+
+ my $repeat = $self->{crm}->do_one_iteration();
+
+ $self->{haenv}->loop_end_hook();
+
+ last if !$repeat;
+ }
+}
+
+sub shutdown {
+ my ($self) = @_;
+
+ $self->{crm}->shutdown_request();
+}
+
+$daemon->register_start_command();
+$daemon->register_stop_command();
+$daemon->register_status_command();
+
+my $cmddef = {
+ start => [ __PACKAGE__, 'start', []],
+ stop => [ __PACKAGE__, 'stop', []],
+ status => [ __PACKAGE__, 'status', [], undef, sub { print shift . "\n";} ],
+};
+
+my $cmd = shift;
+
+PVE::CLIHandler::handle_cmd($cmddef, $0, $cmd, \@ARGV, undef, $0);
+
+exit (0);
+
+__END__
+
+=head1 NAME
+
+pve-ha-crm - PVE Cluster Ressource Manager Daemon
+
+=head1 SYNOPSIS
+
+=include synopsis
+
+=head1 DESCRIPTION
+
+This is the Cluster Ressource Manager.
+
+=include pve_copyright
--- /dev/null
+#!/usr/bin/perl
+
+use strict;
+use warnings;
+use PVE::SafeSyslog;
+use PVE::Daemon;
+use Data::Dumper;
+use PVE::RPCEnvironment;
+
+use PVE::HA::Env;
+use PVE::HA::Env::PVE2;
+use PVE::HA::LRM;
+
+use base qw(PVE::Daemon);
+
+my $cmdline = [$0, @ARGV];
+
+my %daemon_options = (stop_wait_time => 60);
+
+my $daemon = __PACKAGE__->new('pve-ha-lrm', $cmdline, %daemon_options);
+
+my $rpcenv = PVE::RPCEnvironment->init('cli');
+
+$rpcenv->init_request();
+$rpcenv->set_language($ENV{LANG});
+$rpcenv->set_user('root@pam');
+
+sub run {
+ my ($self) = @_;
+
+ $self->{haenv} = PVE::HA::Env->new('PVE::HA::Env::PVE2', $self->{nodename});
+
+ $self->{lrm} = PVE::HA::LRM->new($self->{haenv});
+
+ for (;;) {
+ $self->{haenv}->loop_start_hook();
+
+ my $repeat = $self->{lrm}->do_one_iteration();
+
+ $self->{haenv}->loop_end_hook();
+
+ last if !$repeat;
+ }
+}
+
+sub shutdown {
+ my ($self) = @_;
+
+ $self->{lrm}->shutdown_request();
+}
+
+$daemon->register_start_command();
+$daemon->register_stop_command();
+$daemon->register_status_command();
+
+my $cmddef = {
+ start => [ __PACKAGE__, 'start', []],
+ stop => [ __PACKAGE__, 'stop', []],
+ status => [ __PACKAGE__, 'status', [], undef, sub { print shift . "\n";} ],
+};
+
+my $cmd = shift;
+
+PVE::CLIHandler::handle_cmd($cmddef, $0, $cmd, \@ARGV, undef, $0);
+
+exit (0);
+
+__END__
+
+=head1 NAME
+
+pve-ha-lrm - PVE Local HA Ressource Manager Daemon
+
+=head1 SYNOPSIS
+
+=include synopsis
+
+=head1 DESCRIPTION
+
+This is the Local HA Ressource Manager.
+
+=include pve_copyright
--- /dev/null
+#!/usr/bin/perl
+
+use strict;
+use warnings;
+use lib '/usr/share/pve-ha-simulator';
+use Getopt::Long;
+use JSON;
+
+use PVE::Tools;
+use PVE::HA::Sim::TestHardware;
+use PVE::HA::Sim::RTHardware;
+
+my $opt_batch;
+
+sub show_usage {
+ print "usage: $0 <testdir> [--batch]\n";
+ exit(-1);
+};
+
+if (!GetOptions ("batch" => \$opt_batch)) {
+ show_usage();
+}
+
+my $testdir = shift || show_usage();
+
+my $hardware;
+
+if ($opt_batch) {
+ $hardware = PVE::HA::Sim::TestHardware->new($testdir);
+} else {
+ $hardware = PVE::HA::Sim::RTHardware->new($testdir);
+}
+
+$hardware->log('info', "starting simulation");
+
+eval { $hardware->run(); };
+if (my $err = $@) {
+ $hardware->log('err', "exit simulation - $err ");
+} else {
+ $hardware->log('info', "exit simulation - done");
+}
+
+exit(0);
+
+
+
--- /dev/null
+all:
+
+
+.PHONY: test
+test:
+ echo "running regression tests"
+ ./ha-tester.pl
+ echo "end regression tests (success)"
+
+.PHONY: clean
+clean:
+ rm -rf *~ test-*/log test-*/*~ test-*/status
--- /dev/null
+#!/usr/bin/perl
+
+use strict;
+use warnings;
+use Getopt::Long;
+
+use File::Path qw(make_path remove_tree);
+
+use PVE::Tools;
+
+
+my $opt_nodiff;
+
+if (!GetOptions ("nodiff" => \$opt_nodiff)) {
+ print "usage: $0 testdir [--nodiff]\n";
+ exit -1;
+}
+
+
+#my $testcmd = "../pve-ha-manager --test"
+
+sub run_test {
+ my $dir = shift;
+
+ $dir =~ s!/+$!!;
+
+ print "run: $dir\n";
+
+ my $logfile = "$dir/log";
+ my $logexpect = "$logfile.expect";
+
+ my $res = system("perl -I ../ ../pve-ha-simulator $dir|tee $logfile");
+ die "Test '$dir' failed\n" if $res != 0;
+
+ return if $opt_nodiff;
+
+ if (-f $logexpect) {
+ my $cmd = ['diff', '-u', $logexpect, $logfile];
+ $res = system(@$cmd);
+ die "test '$dir' failed\n" if $res != 0;
+ } else {
+ $res = system('cp', $logfile, $logexpect);
+ die "test '$dir' failed\n" if $res != 0;
+ }
+ print "end: $dir (success)\n";
+}
+
+if (my $testdir = shift) {
+ run_test($testdir);
+} else {
+ foreach my $dir (<test-*>) {
+ run_test($dir);
+ }
+}
+
+
+
--- /dev/null
+#!/bin/sh
+
+rm -rf simtest
+mkdir simtest
+perl -I .. ../pve-ha-simulator simtest
--- /dev/null
+info 0 node1: HA is not enabled
--- /dev/null
+info 0 abc: starting simulation environment (status = wait_for_quorum)
+err 1005 abc: exit now (status = wait_for_quorum) - simulation end
+
--- /dev/null
+{}
\ No newline at end of file
--- /dev/null
+info 0 node1: starting simulation environment (status = wait_for_quorum)
+info 101 node1: manager status change wait_for_quorum => master
+info 101 node1: node 'node2' status change: 'new' => 'online'
+info 101 node1: node 'node1' status change: 'new' => 'online'
+info 200 node1: node 'node3' status change: 'new' => 'online'
+info 310 node1: node 'node3' status change: 'online' => 'unknown'
+info 409 node1: node 'node3' status change: 'unknown' => 'online'
+info 914 node1: manager status change master => wait_for_quorum
+info 1015 node1: manager status change wait_for_quorum => slave
+info 1101 node1: manager status change slave => master
+info 4818 node1: manager status change master => wait_for_quorum
+err 5803 node1: exit now (status = wait_for_quorum) - simulation end
+
--- /dev/null
+{}
\ No newline at end of file
--- /dev/null
+[
+ [ 100 , [ "node1", "node2" ]],
+ [ 200 , [ "node1", "node2", "node3" ]],
+ [ 300 , [ "node1", "node2" ]],
+ [ 400 , [ "node1", "node2", "node3"]],
+ [ 900 , [ "node2", "node3" ]],
+ [ 1000 , [ "node2", "node3", "node1" ]],
+ [ 1100 , [ "node1", "node2", "node3" ]],
+ [ 4800 , [ "node2", "node3" ]]
+]
--- /dev/null
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <string.h>
+#include <sys/ioctl.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+#include <sys/epoll.h>
+
+#include <linux/types.h>
+#include <linux/watchdog.h>
+
+#include <systemd/sd-daemon.h>
+
+#define MY_SOCK_PATH "/run/watchdog-mux.sock"
+#define LISTEN_BACKLOG 50
+#define MAX_EVENTS 10
+
+#define WATCHDOG_DEV "/dev/watchdog"
+
+int watchdog_fd = -1;
+int watchdog_timeout = 20;
+
+static void
+watchdog_close(void)
+{
+ if (watchdog_fd != -1) {
+ if (write(watchdog_fd, "V", 1) == -1) {
+ perror("write magic watchdog close");
+ }
+ if (close(watchdog_fd) == -1) {
+ perror("write magic watchdog close");
+ }
+ }
+
+ watchdog_fd = -1;
+}
+
+int
+main(void)
+{
+ struct sockaddr_un my_addr, peer_addr;
+ socklen_t peer_addr_size;
+ struct epoll_event ev, events[MAX_EVENTS];
+ int socket_count, listen_sock, nfds, epollfd;
+
+ struct stat fs;
+ if (stat(WATCHDOG_DEV, &fs) == -1) {
+ system("modprobe -q softdog soft_noboot=1"); // fixme
+ }
+
+ if ((watchdog_fd = open(WATCHDOG_DEV, O_WRONLY)) == -1) {
+ perror("watchdog open");
+ exit(EXIT_FAILURE);
+ }
+
+ if (ioctl(watchdog_fd, WDIOC_SETTIMEOUT, &watchdog_timeout) == -1) {
+ perror("watchdog set timeout");
+ watchdog_close();
+ exit(EXIT_FAILURE);
+ }
+
+ /* read and log watchdog identity */
+ struct watchdog_info wdinfo;
+ if (ioctl(watchdog_fd, WDIOC_GETSUPPORT, &wdinfo) == -1) {
+ perror("read watchdog info");
+ watchdog_close();
+ exit(EXIT_FAILURE);
+ }
+
+ wdinfo.identity[sizeof(wdinfo.identity) - 1] = 0; // just to be sure
+ fprintf(stderr, "Watchdog driver '%s', version %x\n",
+ wdinfo.identity, wdinfo.firmware_version);
+
+ socket_count = sd_listen_fds(0);
+
+ if (socket_count > 1) {
+
+ perror("too many file descriptors received.\n");
+ goto err;
+
+ } else if (socket_count == 1) {
+
+ listen_sock = SD_LISTEN_FDS_START + 0;
+
+ } else {
+
+ unlink(MY_SOCK_PATH);
+
+ listen_sock = socket(AF_UNIX, SOCK_STREAM, 0);
+ if (listen_sock == -1) {
+ perror("socket create");
+ exit(EXIT_FAILURE);
+ }
+
+ memset(&my_addr, 0, sizeof(struct sockaddr_un));
+ my_addr.sun_family = AF_UNIX;
+ strncpy(my_addr.sun_path, MY_SOCK_PATH, sizeof(my_addr.sun_path) - 1);
+
+ if (bind(listen_sock, (struct sockaddr *) &my_addr,
+ sizeof(struct sockaddr_un)) == -1) {
+ perror("socket bind");
+ exit(EXIT_FAILURE);
+ }
+
+ if (listen(listen_sock, LISTEN_BACKLOG) == -1) {
+ perror("socket listen");
+ goto err;
+ }
+ }
+
+ epollfd = epoll_create(10);
+ if (epollfd == -1) {
+ perror("epoll_create");
+ goto err;
+ }
+
+ ev.events = EPOLLIN;
+ ev.data.fd = listen_sock;
+ if (epoll_ctl(epollfd, EPOLL_CTL_ADD, listen_sock, &ev) == -1) {
+ perror("epoll_ctl: listen_sock");
+ goto err;
+ }
+
+ for (;;) {
+ nfds = epoll_wait(epollfd, events, MAX_EVENTS, -1); //fixme: timeout
+ if (nfds == -1) {
+ perror("epoll_pwait");
+ goto err;
+ }
+
+ int n;
+ for (n = 0; n < nfds; ++n) {
+ if (events[n].data.fd == listen_sock) {
+ int conn_sock = accept(listen_sock, (struct sockaddr *) &peer_addr, &peer_addr_size);
+ if (conn_sock == -1) {
+ perror("accept");
+ goto err; // fixme
+ }
+ if (fcntl(conn_sock, F_SETFL, O_NONBLOCK) == -1) {
+ perror("setnonblocking");
+ goto err; // fixme
+ }
+
+ ev.events = EPOLLIN;
+ ev.data.fd = conn_sock;
+ if (epoll_ctl(epollfd, EPOLL_CTL_ADD, conn_sock, &ev) == -1) {
+ perror("epoll_ctl: add conn_sock");
+ goto err; // fixme
+ }
+ } else {
+ char buf[4096];
+ int cfd = events[n].data.fd;
+
+ ssize_t bytes = read(cfd, buf, sizeof(buf));
+ if (bytes == -1) {
+ perror("read");
+ goto err; // fixme
+ } else if (bytes > 0) {
+ printf("GOT %zd bytes\n", bytes);
+ } else {
+ if (events[n].events & EPOLLHUP || events[n].events & EPOLLERR) {
+ printf("GOT %016x event\n", events[n].events);
+ if (epoll_ctl(epollfd, EPOLL_CTL_DEL, cfd, NULL) == -1) {
+ perror("epoll_ctl: del conn_sock");
+ goto err; // fixme
+ }
+ if (close(cfd) == -1) {
+ perror("close conn_sock");
+ goto err; // fixme
+ }
+ }
+ }
+ }
+ }
+ }
+
+ printf("DONE\n");
+
+// out:
+
+ watchdog_close();
+ unlink(MY_SOCK_PATH);
+ exit(EXIT_SUCCESS);
+
+err:
+ unlink(MY_SOCK_PATH);
+ exit(EXIT_FAILURE);
+}
+++ /dev/null
-all:
-
-
-.PHONY: test
-test:
- echo "running regression tests"
- ./ha-tester.pl
- echo "end regression tests (success)"
-
-.PHONY: clean
-clean:
- rm -rf *~ test-*/log test-*/*~ test-*/status
+++ /dev/null
-#!/usr/bin/perl
-
-use strict;
-use warnings;
-use Getopt::Long;
-
-use File::Path qw(make_path remove_tree);
-
-use PVE::Tools;
-
-
-my $opt_nodiff;
-
-if (!GetOptions ("nodiff" => \$opt_nodiff)) {
- print "usage: $0 testdir [--nodiff]\n";
- exit -1;
-}
-
-
-#my $testcmd = "../pve-ha-manager --test"
-
-sub run_test {
- my $dir = shift;
-
- $dir =~ s!/+$!!;
-
- print "run: $dir\n";
-
- my $logfile = "$dir/log";
- my $logexpect = "$logfile.expect";
-
- my $res = system("perl -I ../ ../pve-ha-simulator $dir|tee $logfile");
- die "Test '$dir' failed\n" if $res != 0;
-
- return if $opt_nodiff;
-
- if (-f $logexpect) {
- my $cmd = ['diff', '-u', $logexpect, $logfile];
- $res = system(@$cmd);
- die "test '$dir' failed\n" if $res != 0;
- } else {
- $res = system('cp', $logfile, $logexpect);
- die "test '$dir' failed\n" if $res != 0;
- }
- print "end: $dir (success)\n";
-}
-
-if (my $testdir = shift) {
- run_test($testdir);
-} else {
- foreach my $dir (<test-*>) {
- run_test($dir);
- }
-}
-
-
-
+++ /dev/null
-#!/bin/sh
-
-rm -rf simtest
-mkdir simtest
-perl -I .. ../pve-ha-simulator simtest
+++ /dev/null
-info 0 node1: HA is not enabled
+++ /dev/null
-info 0 abc: starting simulation environment (status = wait_for_quorum)
-err 1005 abc: exit now (status = wait_for_quorum) - simulation end
-
+++ /dev/null
-{}
\ No newline at end of file
+++ /dev/null
-info 0 node1: starting simulation environment (status = wait_for_quorum)
-info 101 node1: manager status change wait_for_quorum => master
-info 101 node1: node 'node2' status change: 'new' => 'online'
-info 101 node1: node 'node1' status change: 'new' => 'online'
-info 200 node1: node 'node3' status change: 'new' => 'online'
-info 310 node1: node 'node3' status change: 'online' => 'unknown'
-info 409 node1: node 'node3' status change: 'unknown' => 'online'
-info 914 node1: manager status change master => wait_for_quorum
-info 1015 node1: manager status change wait_for_quorum => slave
-info 1101 node1: manager status change slave => master
-info 4818 node1: manager status change master => wait_for_quorum
-err 5803 node1: exit now (status = wait_for_quorum) - simulation end
-
+++ /dev/null
-{}
\ No newline at end of file
+++ /dev/null
-[
- [ 100 , [ "node1", "node2" ]],
- [ 200 , [ "node1", "node2", "node3" ]],
- [ 300 , [ "node1", "node2" ]],
- [ 400 , [ "node1", "node2", "node3"]],
- [ 900 , [ "node2", "node3" ]],
- [ 1000 , [ "node2", "node3", "node1" ]],
- [ 1100 , [ "node1", "node2", "node3" ]],
- [ 4800 , [ "node2", "node3" ]]
-]
+++ /dev/null
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <fcntl.h>
-#include <string.h>
-#include <sys/ioctl.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <sys/socket.h>
-#include <sys/un.h>
-#include <sys/epoll.h>
-
-#include <linux/types.h>
-#include <linux/watchdog.h>
-
-#include <systemd/sd-daemon.h>
-
-#define MY_SOCK_PATH "/var/run/pve_watchdog"
-#define LISTEN_BACKLOG 50
-#define MAX_EVENTS 10
-
-#define WATCHDOG_DEV "/dev/watchdog"
-
-int watchdog_fd = -1;
-int watchdog_timeout = 20;
-
-static void
-watchdog_close(void)
-{
- if (watchdog_fd != -1) {
- if (write(watchdog_fd, "V", 1) == -1) {
- perror("write magic watchdog close");
- }
- if (close(watchdog_fd) == -1) {
- perror("write magic watchdog close");
- }
- }
-
- watchdog_fd = -1;
-}
-
-int
-main(void)
-{
- struct sockaddr_un my_addr, peer_addr;
- socklen_t peer_addr_size;
- struct epoll_event ev, events[MAX_EVENTS];
- int socket_count, listen_sock, nfds, epollfd;
-
- struct stat fs;
- if (stat(WATCHDOG_DEV, &fs) == -1) {
- system("modprobe -q softdog soft_noboot=1"); // fixme
- }
-
- if ((watchdog_fd = open(WATCHDOG_DEV, O_WRONLY)) == -1) {
- perror("watchdog open");
- exit(EXIT_FAILURE);
- }
-
- if (ioctl(watchdog_fd, WDIOC_SETTIMEOUT, &watchdog_timeout) == -1) {
- perror("watchdog set timeout");
- watchdog_close();
- exit(EXIT_FAILURE);
- }
-
- /* read and log watchdog identity */
- struct watchdog_info wdinfo;
- if (ioctl(watchdog_fd, WDIOC_GETSUPPORT, &wdinfo) == -1) {
- perror("read watchdog info");
- watchdog_close();
- exit(EXIT_FAILURE);
- }
-
- wdinfo.identity[sizeof(wdinfo.identity) - 1] = 0; // just to be sure
- fprintf(stderr, "Watchdog driver '%s', version %x\n",
- wdinfo.identity, wdinfo.firmware_version);
-
- socket_count = sd_listen_fds(0);
-
- if (socket_count > 1) {
-
- perror("Too many file descriptors received.\n");
- goto err;
-
- } else if (socket_count == 1) {
-
- listen_sock = SD_LISTEN_FDS_START + 0;
-
- } else {
-
- unlink(MY_SOCK_PATH);
-
- listen_sock = socket(AF_UNIX, SOCK_STREAM, 0);
- if (listen_sock == -1) {
- perror("socket create");
- exit(EXIT_FAILURE);
- }
-
- memset(&my_addr, 0, sizeof(struct sockaddr_un));
- my_addr.sun_family = AF_UNIX;
- strncpy(my_addr.sun_path, MY_SOCK_PATH, sizeof(my_addr.sun_path) - 1);
-
- if (bind(listen_sock, (struct sockaddr *) &my_addr,
- sizeof(struct sockaddr_un)) == -1) {
- perror("socket bind");
- exit(EXIT_FAILURE);
- }
-
- if (listen(listen_sock, LISTEN_BACKLOG) == -1) {
- perror("socket listen");
- goto err;
- }
- }
-
- epollfd = epoll_create(10);
- if (epollfd == -1) {
- perror("epoll_create");
- goto err;
- }
-
- ev.events = EPOLLIN;
- ev.data.fd = listen_sock;
- if (epoll_ctl(epollfd, EPOLL_CTL_ADD, listen_sock, &ev) == -1) {
- perror("epoll_ctl: listen_sock");
- goto err;
- }
-
- for (;;) {
- nfds = epoll_wait(epollfd, events, MAX_EVENTS, -1); //fixme: timeout
- if (nfds == -1) {
- perror("epoll_pwait");
- goto err;
- }
-
- int n;
- for (n = 0; n < nfds; ++n) {
- if (events[n].data.fd == listen_sock) {
- int conn_sock = accept(listen_sock, (struct sockaddr *) &peer_addr, &peer_addr_size);
- if (conn_sock == -1) {
- perror("accept");
- goto err; // fixme
- }
- if (fcntl(conn_sock, F_SETFL, O_NONBLOCK) == -1) {
- perror("setnonblocking");
- goto err; // fixme
- }
-
- ev.events = EPOLLIN;
- ev.data.fd = conn_sock;
- if (epoll_ctl(epollfd, EPOLL_CTL_ADD, conn_sock, &ev) == -1) {
- perror("epoll_ctl: add conn_sock");
- goto err; // fixme
- }
- } else {
- char buf[4096];
- int cfd = events[n].data.fd;
-
- ssize_t bytes = read(cfd, buf, sizeof(buf));
- if (bytes == -1) {
- perror("read");
- goto err; // fixme
- } else if (bytes > 0) {
- printf("GOT %zd bytes\n", bytes);
- } else {
- if (events[n].events & EPOLLHUP || events[n].events & EPOLLERR) {
- printf("GOT %016x event\n", events[n].events);
- if (epoll_ctl(epollfd, EPOLL_CTL_DEL, cfd, NULL) == -1) {
- perror("epoll_ctl: del conn_sock");
- goto err; // fixme
- }
- if (close(cfd) == -1) {
- perror("close conn_sock");
- goto err; // fixme
- }
- }
- }
- }
- }
- }
-
- printf("DONE\n");
-
-// out:
-
- watchdog_close();
- unlink(MY_SOCK_PATH);
- exit(EXIT_SUCCESS);
-
-err:
- unlink(MY_SOCK_PATH);
- exit(EXIT_FAILURE);
-}
projects / pve-ha-manager.git / commitdiff