src/PVE/HA/Env/PVE2.pm

   1 package PVE::HA::Env::PVE2;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX qw(:errno_h :fcntl_h);
   6 use IO::File;
   7 use IO::Socket::UNIX;
   8
   9 use PVE::SafeSyslog;
  10 use PVE::Tools;
  11 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  12 use PVE::INotify;
  13 use PVE::RPCEnvironment;
  14
  15 use PVE::HA::Tools ':exit_codes';
  16 use PVE::HA::Env;
  17 use PVE::HA::Config;
  18
  19
  20 my $lockdir = "/etc/pve/priv/lock";
  21
  22 sub new {
  23     my ($this, $nodename) = @_;
  24
  25     die "missing nodename" if !$nodename;
  26
  27     my $class = ref($this) || $this;
  28
  29     my $self = bless {}, $class;
  30
  31     $self->{nodename} = $nodename;
  32
  33     return $self;
  34 }
  35
  36 sub nodename {
  37     my ($self) = @_;
  38
  39     return $self->{nodename};
  40 }
  41
  42 sub read_manager_status {
  43     my ($self) = @_;
  44
  45     return PVE::HA::Config::read_manager_status();
  46 }
  47
  48 sub write_manager_status {
  49     my ($self, $status_obj) = @_;
  50
  51     PVE::HA::Config::write_manager_status($status_obj);
  52 }
  53
  54 sub read_lrm_status {
  55     my ($self, $node) = @_;
  56
  57     $node = $self->{nodename} if !defined($node);
  58
  59     return PVE::HA::Config::read_lrm_status($node);
  60 }
  61
  62 sub write_lrm_status {
  63     my ($self, $status_obj) = @_;
  64
  65     my $node = $self->{nodename};
  66
  67     PVE::HA::Config::write_lrm_status($node, $status_obj);
  68 }
  69
  70 sub is_node_shutdown {
  71     my ($self) = @_;
  72
  73     my $shutdown = 0;
  74
  75     my $code = sub {
  76         my $line = shift;
  77
  78         $shutdown = 1 if ($line =~ m/shutdown\.target/);
  79     };
  80
  81     my $cmd = ['/bin/systemctl', 'list-jobs'];
  82     eval { PVE::Tools::run_command($cmd, outfunc => $code, noerr => 1); };
  83
  84     return $shutdown;
  85 }
  86
  87 sub queue_crm_commands {
  88     my ($self, $cmd) = @_;
  89
  90     return PVE::HA::Config::queue_crm_commands($cmd);
  91 }
  92
  93 sub read_crm_commands {
  94     my ($self) = @_;
  95
  96     return PVE::HA::Config::read_crm_commands();
  97 }
  98
  99 sub service_config_exists {
 100     my ($self) = @_;
 101
 102     return PVE::HA::Config::resources_config_exists();
 103 }
 104
 105 sub read_service_config {
 106     my ($self) = @_;
 107
 108     my $res = PVE::HA::Config::read_resources_config();
 109
 110     my $vmlist = PVE::Cluster::get_vmlist();
 111     my $conf = {};
 112
 113     foreach my $sid (keys %{$res->{ids}}) {
 114         my $d = $res->{ids}->{$sid};
 115         my (undef, undef, $name) = PVE::HA::Tools::parse_sid($sid);
 116         $d->{state} = 'enabled' if !defined($d->{state});
 117         $d->{max_restart} = 1 if !defined($d->{max_restart});
 118         $d->{max_relocate} = 1 if !defined($d->{max_relocate});
 119         if (PVE::HA::Resources->lookup($d->{type})) {
 120             if (my $vmd = $vmlist->{ids}->{$name}) {
 121                 if (!$vmd) {
 122                     warn "no such VM '$name'\n";
 123                 } else {
 124                     $d->{node} = $vmd->{node};
 125                     $conf->{$sid} = $d;
 126                 }
 127             } else {
 128                 if (defined($d->{node})) {
 129                     $conf->{$sid} = $d;
 130                 } else {
 131                     warn "service '$sid' without node\n";
 132                 }
 133             }
 134         }
 135     }
 136
 137     return $conf;
 138 }
 139
 140 sub change_service_location {
 141     my ($self, $sid, $current_node, $new_node) = @_;
 142
 143     my (undef, $type, $name) = PVE::HA::Tools::parse_sid($sid);
 144
 145     if(my $plugin = PVE::HA::Resources->lookup($type)) {
 146         my $old = $plugin->config_file($name, $current_node);
 147         my $new = $plugin->config_file($name, $new_node);
 148         rename($old, $new) ||
 149             die "rename '$old' to '$new' failed - $!\n";
 150     } else {
 151         die "implement me";
 152     }
 153 }
 154
 155 sub read_group_config {
 156     my ($self) = @_;
 157
 158     return PVE::HA::Config::read_group_config();
 159 }
 160
 161 # this should return a hash containing info
 162 # what nodes are members and online.
 163 sub get_node_info {
 164     my ($self) = @_;
 165
 166     my ($node_info, $quorate) = ({}, 0);
 167
 168     my $nodename = $self->{nodename};
 169
 170     $quorate = PVE::Cluster::check_cfs_quorum(1) || 0;
 171
 172     my $members = PVE::Cluster::get_members();
 173
 174     foreach my $node (keys %$members) {
 175         my $d = $members->{$node};
 176         $node_info->{$node}->{online} = $d->{online};
 177     }
 178
 179     $node_info->{$nodename}->{online} = 1; # local node is always up
 180
 181     return ($node_info, $quorate);
 182 }
 183
 184 sub log {
 185     my ($self, $level, $msg) = @_;
 186
 187     chomp $msg;
 188
 189     syslog($level, $msg);
 190 }
 191
 192 my $last_lock_status = {};
 193
 194 sub get_pve_lock {
 195     my ($self, $lockid) = @_;
 196
 197     my $got_lock = 0;
 198
 199     my $filename = "$lockdir/$lockid";
 200
 201     my $last = $last_lock_status->{$lockid} || 0;
 202
 203     my $ctime = time();
 204
 205     my $retry = 0;
 206     my $retry_timeout = 100; # fixme: what timeout
 207
 208     eval {
 209
 210         mkdir $lockdir;
 211
 212         # pve cluster filesystem not online
 213         die "can't create '$lockdir' (pmxcfs not mounted?)\n" if ! -d $lockdir;
 214
 215         if ($last && (($ctime - $last) < $retry_timeout)) {
 216              # send cfs lock update request (utime)
 217             if (!utime(0, $ctime, $filename))  {
 218                 $retry = 1;
 219                 die "cfs lock update failed - $!\n";
 220             }
 221         } else {
 222
 223             # fixme: wait some time?
 224             if (!(mkdir $filename)) {
 225                 utime 0, 0, $filename; # cfs unlock request
 226                 die "can't get cfs lock\n";
 227             }
 228         }
 229
 230         $got_lock = 1;
 231     };
 232
 233     my $err = $@;
 234
 235     if ($retry) {
 236         # $self->log('err', $err) if $err; # for debugging
 237         return 0;
 238     }
 239
 240     $last_lock_status->{$lockid} = $got_lock ? $ctime : 0;
 241
 242     if (!!$got_lock != !!$last) {
 243         if ($got_lock) {
 244             $self->log('info', "successfully acquired lock '$lockid'");
 245         } else {
 246             my $msg = "lost lock '$lockid";
 247             $msg .= " - $err" if $err;
 248             $self->log('err', $msg);
 249         }
 250     } else {
 251         # $self->log('err', $err) if $err; # for debugging
 252     }
 253
 254     return $got_lock;
 255 }
 256
 257 sub get_ha_manager_lock {
 258     my ($self) = @_;
 259
 260     return $self->get_pve_lock("ha_manager_lock");
 261 }
 262
 263 sub get_ha_agent_lock {
 264     my ($self, $node) = @_;
 265
 266     $node = $self->nodename() if !defined($node);
 267
 268     return $self->get_pve_lock("ha_agent_${node}_lock");
 269 }
 270
 271 sub quorate {
 272     my ($self) = @_;
 273
 274     my $quorate = 0;
 275     eval {
 276         $quorate = PVE::Cluster::check_cfs_quorum();
 277     };
 278
 279     return $quorate;
 280 }
 281
 282 sub get_time {
 283     my ($self) = @_;
 284
 285     return time();
 286 }
 287
 288 sub sleep {
 289     my ($self, $delay) = @_;
 290
 291     CORE::sleep($delay);
 292 }
 293
 294 sub sleep_until {
 295    my ($self, $end_time) = @_;
 296
 297    for (;;) {
 298        my $cur_time = time();
 299
 300        last if $cur_time >= $end_time;
 301
 302        $self->sleep(1);
 303    }
 304 }
 305
 306 sub loop_start_hook {
 307     my ($self) = @_;
 308
 309     PVE::Cluster::cfs_update();
 310
 311     $self->{loop_start} = $self->get_time();
 312 }
 313
 314 sub loop_end_hook {
 315     my ($self) = @_;
 316
 317     my $delay = $self->get_time() - $self->{loop_start};
 318
 319     warn "loop take too long ($delay seconds)\n" if $delay > 30;
 320 }
 321
 322 my $watchdog_fh;
 323
 324 sub watchdog_open {
 325     my ($self) = @_;
 326
 327     die "watchdog already open\n" if defined($watchdog_fh);
 328
 329     $watchdog_fh = IO::Socket::UNIX->new(
 330         Type => SOCK_STREAM(),
 331         Peer => "/run/watchdog-mux.sock") ||
 332         die "unable to open watchdog socket - $!\n";
 333
 334     $self->log('info', "watchdog active");
 335 }
 336
 337 sub watchdog_update {
 338     my ($self, $wfh) = @_;
 339
 340     my $res = $watchdog_fh->syswrite("\0", 1);
 341     if (!defined($res)) {
 342         $self->log('err', "watchdog update failed - $!\n");
 343         return 0;
 344     }
 345     if ($res != 1) {
 346         $self->log('err', "watchdog update failed - write $res bytes\n");
 347         return 0;
 348     }
 349
 350     return 1;
 351 }
 352
 353 sub watchdog_close {
 354     my ($self, $wfh) = @_;
 355
 356     $watchdog_fh->syswrite("V", 1); # magic watchdog close
 357     if (!$watchdog_fh->close()) {
 358         $self->log('err', "watchdog close failed - $!");
 359     } else {
 360         $watchdog_fh = undef;
 361         $self->log('info', "watchdog closed (disabled)");
 362     }
 363 }
 364
 365 sub upid_wait {
 366     my ($self, $upid) = @_;
 367
 368     my $task = PVE::Tools::upid_decode($upid);
 369
 370     CORE::sleep(1);
 371     while (PVE::ProcFSTools::check_process_running($task->{pid}, $task->{pstart})) {
 372         $self->log('debug', "Task still active, waiting");
 373         CORE::sleep(1);
 374     }
 375 }
 376
 377 sub can_fork {
 378     my ($self) = @_;
 379
 380     return 1;
 381 }
 382
 383 sub exec_resource_agent {
 384     my ($self, $sid, $service_config, $cmd, @params) = @_;
 385
 386     # setup execution environment
 387
 388     $ENV{'PATH'} = '/sbin:/bin:/usr/sbin:/usr/bin';
 389
 390     PVE::INotify::inotify_close();
 391
 392     PVE::INotify::inotify_init();
 393
 394     PVE::Cluster::cfs_update();
 395
 396     my $nodename = $self->{nodename};
 397
 398     my (undef, $service_type, $service_name) = PVE::HA::Tools::parse_sid($sid);
 399
 400     my $plugin = PVE::HA::Resources->lookup($service_type);
 401     if (!$plugin) {
 402         $self->log('err', "service type '$service_type' not implemented");
 403         return EUNKNOWN_SERVICE_TYPE;
 404     }
 405
 406     if ($service_config->{node} ne $nodename) {
 407         $self->log('err', "service '$sid' not on this node");
 408         return EWRONG_NODE;
 409     }
 410
 411     my $vmid = $service_name;
 412
 413     my $running = $plugin->check_running($vmid);
 414
 415     if ($cmd eq 'started') {
 416
 417         return SUCCESS if $running;
 418
 419         $self->log("info", "starting service $sid");
 420
 421         $plugin->start($self, $vmid);
 422
 423         $running = $plugin->check_running($vmid);
 424
 425         if ($running) {
 426             $self->log("info", "service status $sid started");
 427             return SUCCESS;
 428         } else {
 429             $self->log("warning", "unable to start service $sid");
 430             return ERROR;
 431         }
 432
 433     } elsif ($cmd eq 'request_stop' || $cmd eq 'stopped') {
 434
 435         return SUCCESS if !$running;
 436
 437         $self->log("info", "stopping service $sid");
 438
 439         $plugin->shutdown($self, $vmid);
 440
 441         $running = $plugin->check_running($vmid);
 442
 443         if (!$running) {
 444             $self->log("info", "service status $sid stopped");
 445             return SUCCESS;
 446         } else {
 447             $self->log("info", "unable to stop stop service $sid (still running)");
 448             return ERROR;
 449         }
 450
 451     } elsif ($cmd eq 'migrate' || $cmd eq 'relocate') {
 452
 453         my $target = $params[0];
 454         if (!defined($target)) {
 455             die "$cmd '$sid' failed - missing target\n" if !defined($target);
 456             return EINVALID_PARAMETER;
 457         }
 458
 459         if ($service_config->{node} eq $target) {
 460             # already there
 461             return SUCCESS;
 462         }
 463
 464         my $online = ($cmd eq 'migrate') ? 1 : 0;
 465
 466         my $oldconfig = $plugin->config_file($vmid, $nodename);
 467
 468         $plugin->migrate($self, $vmid, $target, $online);
 469
 470         # something went wrong if old config file is still there
 471         if (-f $oldconfig) {
 472             $self->log("err", "service $sid not moved (migration error)");
 473             return ERROR;
 474         }
 475
 476         return SUCCESS;
 477
 478     } elsif ($cmd eq 'error') {
 479
 480         if ($running) {
 481             $self->log("err", "service $sid is in an error state while running");
 482         } else {
 483             $self->log("warning", "service $sid is not running and in an error state");
 484         }
 485         return SUCCESS; # error always succeeds
 486
 487     }
 488
 489     $self->log("err", "implement me (cmd '$cmd')");
 490     return EUNKNOWN_COMMAND;
 491 }
 492
 493 1;