src/PVE/HA/Env/PVE2.pm

   1 package PVE::HA::Env::PVE2;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX qw(:errno_h :fcntl_h);
   6 use IO::File;
   7 use IO::Socket::UNIX;
   8
   9 use PVE::SafeSyslog;
  10 use PVE::Tools;
  11 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  12 use PVE::INotify;
  13 use PVE::RPCEnvironment;
  14
  15 use PVE::HA::Tools ':exit_codes';
  16 use PVE::HA::Env;
  17 use PVE::HA::Config;
  18
  19
  20 my $lockdir = "/etc/pve/priv/lock";
  21
  22 sub new {
  23     my ($this, $nodename) = @_;
  24
  25     die "missing nodename" if !$nodename;
  26
  27     my $class = ref($this) || $this;
  28
  29     my $self = bless {}, $class;
  30
  31     $self->{nodename} = $nodename;
  32
  33     return $self;
  34 }
  35
  36 sub nodename {
  37     my ($self) = @_;
  38
  39     return $self->{nodename};
  40 }
  41
  42 sub read_manager_status {
  43     my ($self) = @_;
  44
  45     return PVE::HA::Config::read_manager_status();
  46 }
  47
  48 sub write_manager_status {
  49     my ($self, $status_obj) = @_;
  50
  51     PVE::HA::Config::write_manager_status($status_obj);
  52 }
  53
  54 sub read_lrm_status {
  55     my ($self, $node) = @_;
  56
  57     $node = $self->{nodename} if !defined($node);
  58
  59     return PVE::HA::Config::read_lrm_status($node);
  60 }
  61
  62 sub write_lrm_status {
  63     my ($self, $status_obj) = @_;
  64
  65     my $node = $self->{nodename};
  66
  67     PVE::HA::Config::write_lrm_status($node, $status_obj);
  68 }
  69
  70 sub queue_crm_commands {
  71     my ($self, $cmd) = @_;
  72
  73     return PVE::HA::Config::queue_crm_commands($cmd);
  74 }
  75
  76 sub read_crm_commands {
  77     my ($self) = @_;
  78
  79     return PVE::HA::Config::read_crm_commands();
  80 }
  81
  82 sub service_config_exists {
  83     my ($self) = @_;
  84
  85     return PVE::HA::Config::resources_config_exists();
  86 }
  87
  88 sub read_service_config {
  89     my ($self) = @_;
  90
  91     my $res = PVE::HA::Config::read_resources_config();
  92
  93     my $vmlist = PVE::Cluster::get_vmlist();
  94     my $conf = {};
  95
  96     foreach my $sid (keys %{$res->{ids}}) {
  97         my $d = $res->{ids}->{$sid};
  98         my (undef, undef, $name) = PVE::HA::Tools::parse_sid($sid);
  99         $d->{state} = 'enabled' if !defined($d->{state});
 100         $d->{max_restart} = 1 if !defined($d->{max_restart});
 101         $d->{max_relocate} = 1 if !defined($d->{max_relocate});
 102         if (PVE::HA::Resources->lookup($d->{type})) {
 103             if (my $vmd = $vmlist->{ids}->{$name}) {
 104                 if (!$vmd) {
 105                     warn "no such VM '$name'\n";
 106                 } else {
 107                     $d->{node} = $vmd->{node};
 108                     $conf->{$sid} = $d;
 109                 }
 110             } else {
 111                 if (defined($d->{node})) {
 112                     $conf->{$sid} = $d;
 113                 } else {
 114                     warn "service '$sid' without node\n";
 115                 }
 116             }
 117         }
 118     }
 119
 120     return $conf;
 121 }
 122
 123 sub change_service_location {
 124     my ($self, $sid, $current_node, $new_node) = @_;
 125
 126     my (undef, $type, $name) = PVE::HA::Tools::parse_sid($sid);
 127
 128     if(my $plugin = PVE::HA::Resources->lookup($type)) {
 129         my $old = $plugin->config_file($name, $current_node);
 130         my $new = $plugin->config_file($name, $new_node);
 131         rename($old, $new) ||
 132             die "rename '$old' to '$new' failed - $!\n";
 133     } else {
 134         die "implement me";
 135     }
 136 }
 137
 138 sub read_group_config {
 139     my ($self) = @_;
 140
 141     return PVE::HA::Config::read_group_config();
 142 }
 143
 144 # this should return a hash containing info
 145 # what nodes are members and online.
 146 sub get_node_info {
 147     my ($self) = @_;
 148
 149     my ($node_info, $quorate) = ({}, 0);
 150
 151     my $nodename = $self->{nodename};
 152
 153     $quorate = PVE::Cluster::check_cfs_quorum(1) || 0;
 154
 155     my $members = PVE::Cluster::get_members();
 156
 157     foreach my $node (keys %$members) {
 158         my $d = $members->{$node};
 159         $node_info->{$node}->{online} = $d->{online};
 160     }
 161
 162     $node_info->{$nodename}->{online} = 1; # local node is always up
 163
 164     return ($node_info, $quorate);
 165 }
 166
 167 sub log {
 168     my ($self, $level, $msg) = @_;
 169
 170     chomp $msg;
 171
 172     syslog($level, $msg);
 173 }
 174
 175 my $last_lock_status = {};
 176
 177 sub get_pve_lock {
 178     my ($self, $lockid) = @_;
 179
 180     my $got_lock = 0;
 181
 182     my $filename = "$lockdir/$lockid";
 183
 184     my $last = $last_lock_status->{$lockid} || 0;
 185
 186     my $ctime = time();
 187
 188     my $retry = 0;
 189     my $retry_timeout = 100; # fixme: what timeout
 190
 191     eval {
 192
 193         mkdir $lockdir;
 194
 195         # pve cluster filesystem not online
 196         die "can't create '$lockdir' (pmxcfs not mounted?)\n" if ! -d $lockdir;
 197
 198         if ($last && (($ctime - $last) < $retry_timeout)) {
 199              # send cfs lock update request (utime)
 200             if (!utime(0, $ctime, $filename))  {
 201                 $retry = 1;
 202                 die "cfs lock update failed - $!\n";
 203             }
 204         } else {
 205
 206             # fixme: wait some time?
 207             if (!(mkdir $filename)) {
 208                 utime 0, 0, $filename; # cfs unlock request
 209                 die "can't get cfs lock\n";
 210             }
 211         }
 212
 213         $got_lock = 1;
 214     };
 215
 216     my $err = $@;
 217
 218     if ($retry) {
 219         # $self->log('err', $err) if $err; # for debugging
 220         return 0;
 221     }
 222
 223     $last_lock_status->{$lockid} = $got_lock ? $ctime : 0;
 224
 225     if (!!$got_lock != !!$last) {
 226         if ($got_lock) {
 227             $self->log('info', "successfully acquired lock '$lockid'");
 228         } else {
 229             my $msg = "lost lock '$lockid";
 230             $msg .= " - $err" if $err;
 231             $self->log('err', $msg);
 232         }
 233     } else {
 234         # $self->log('err', $err) if $err; # for debugging
 235     }
 236
 237     return $got_lock;
 238 }
 239
 240 sub get_ha_manager_lock {
 241     my ($self) = @_;
 242
 243     return $self->get_pve_lock("ha_manager_lock");
 244 }
 245
 246 sub get_ha_agent_lock {
 247     my ($self, $node) = @_;
 248
 249     $node = $self->nodename() if !defined($node);
 250
 251     return $self->get_pve_lock("ha_agent_${node}_lock");
 252 }
 253
 254 sub quorate {
 255     my ($self) = @_;
 256
 257     my $quorate = 0;
 258     eval {
 259         $quorate = PVE::Cluster::check_cfs_quorum();
 260     };
 261
 262     return $quorate;
 263 }
 264
 265 sub get_time {
 266     my ($self) = @_;
 267
 268     return time();
 269 }
 270
 271 sub sleep {
 272     my ($self, $delay) = @_;
 273
 274     CORE::sleep($delay);
 275 }
 276
 277 sub sleep_until {
 278    my ($self, $end_time) = @_;
 279
 280    for (;;) {
 281        my $cur_time = time();
 282
 283        last if $cur_time >= $end_time;
 284
 285        $self->sleep(1);
 286    }
 287 }
 288
 289 sub loop_start_hook {
 290     my ($self) = @_;
 291
 292     PVE::Cluster::cfs_update();
 293
 294     $self->{loop_start} = $self->get_time();
 295 }
 296
 297 sub loop_end_hook {
 298     my ($self) = @_;
 299
 300     my $delay = $self->get_time() - $self->{loop_start};
 301
 302     warn "loop take too long ($delay seconds)\n" if $delay > 30;
 303 }
 304
 305 my $watchdog_fh;
 306
 307 sub watchdog_open {
 308     my ($self) = @_;
 309
 310     die "watchdog already open\n" if defined($watchdog_fh);
 311
 312     $watchdog_fh = IO::Socket::UNIX->new(
 313         Type => SOCK_STREAM(),
 314         Peer => "/run/watchdog-mux.sock") ||
 315         die "unable to open watchdog socket - $!\n";
 316
 317     $self->log('info', "watchdog active");
 318 }
 319
 320 sub watchdog_update {
 321     my ($self, $wfh) = @_;
 322
 323     my $res = $watchdog_fh->syswrite("\0", 1);
 324     if (!defined($res)) {
 325         $self->log('err', "watchdog update failed - $!\n");
 326         return 0;
 327     }
 328     if ($res != 1) {
 329         $self->log('err', "watchdog update failed - write $res bytes\n");
 330         return 0;
 331     }
 332
 333     return 1;
 334 }
 335
 336 sub watchdog_close {
 337     my ($self, $wfh) = @_;
 338
 339     $watchdog_fh->syswrite("V", 1); # magic watchdog close
 340     if (!$watchdog_fh->close()) {
 341         $self->log('err', "watchdog close failed - $!");
 342     } else {
 343         $watchdog_fh = undef;
 344         $self->log('info', "watchdog closed (disabled)");
 345     }
 346 }
 347
 348 sub upid_wait {
 349     my ($self, $upid) = @_;
 350
 351     my $task = PVE::Tools::upid_decode($upid);
 352
 353     CORE::sleep(1);
 354     while (PVE::ProcFSTools::check_process_running($task->{pid}, $task->{pstart})) {
 355         $self->log('debug', "Task still active, waiting");
 356         CORE::sleep(1);
 357     }
 358 }
 359
 360 sub can_fork {
 361     my ($self) = @_;
 362
 363     return 1;
 364 }
 365
 366 sub exec_resource_agent {
 367     my ($self, $sid, $service_config, $cmd, @params) = @_;
 368
 369     # setup execution environment
 370
 371     $ENV{'PATH'} = '/sbin:/bin:/usr/sbin:/usr/bin';
 372
 373     PVE::INotify::inotify_close();
 374
 375     PVE::INotify::inotify_init();
 376
 377     PVE::Cluster::cfs_update();
 378
 379     my $nodename = $self->{nodename};
 380
 381     # fixme: return valid_exit code (instead of using die) ?
 382
 383     my (undef, $service_type, $service_name) = PVE::HA::Tools::parse_sid($sid);
 384
 385     my $plugin = PVE::HA::Resources->lookup($service_type);
 386     die "service type '$service_type' not implemented" if !$plugin;
 387
 388     # fixme: return valid_exit code
 389     die "service '$sid' not on this node" if $service_config->{node} ne $nodename;
 390
 391     my $vmid = $service_name;
 392
 393     my $running = $plugin->check_running($vmid);
 394
 395     if ($cmd eq 'started') {
 396
 397         return SUCCESS if $running;
 398
 399         $self->log("info", "starting service $sid");
 400
 401         my $params = {
 402             node => $nodename,
 403             vmid => $vmid
 404         };
 405
 406         $plugin->start($self, $params);
 407
 408         $running = $plugin->check_running($vmid);
 409
 410         if ($running) {
 411             $self->log("info", "service status $sid started");
 412             return SUCCESS;
 413         } else {
 414             $self->log("warning", "unable to start service $sid");
 415             return ERROR;
 416         }
 417
 418     } elsif ($cmd eq 'request_stop' || $cmd eq 'stopped') {
 419
 420         return SUCCESS if !$running;
 421
 422         $self->log("info", "stopping service $sid");
 423
 424         my $timeout = 60; # fixme: make this configurable
 425
 426         my $params = {
 427             node => $nodename,
 428             vmid => $vmid,
 429             timeout => $timeout,
 430             forceStop => 1,
 431         };
 432
 433         $plugin->shutdown($self, $params);
 434
 435         $running = $plugin->check_running($vmid);
 436
 437         if (!$running) {
 438             $self->log("info", "service status $sid stopped");
 439             return SUCCESS;
 440         } else {
 441             return ERROR;
 442         }
 443
 444     } elsif ($cmd eq 'migrate' || $cmd eq 'relocate') {
 445
 446         my $target = $params[0];
 447         die "$cmd '$sid' failed - missing target\n" if !defined($target);
 448
 449         if ($service_config->{node} eq $target) {
 450             # already there
 451             return SUCCESS;
 452         }
 453
 454         # we always do (live) migration
 455         my $params = {
 456             node => $nodename,
 457             vmid => $vmid,
 458             target => $target,
 459             online => 1,
 460         };
 461
 462         my $oldconfig = $plugin->config_file($vmid, $nodename);
 463
 464         $plugin->migrate($self, $params);
 465
 466         # something went wrong if old config file is still there
 467         if (-f $oldconfig) {
 468             $self->log("err", "service $sid not moved (migration error)");
 469             return ERROR;
 470         }
 471
 472         return SUCCESS;
 473
 474     } elsif ($cmd eq 'error') {
 475
 476         if ($running) {
 477             $self->log("err", "service $sid is in an error state while running");
 478         } else {
 479             $self->log("warning", "service $sid is not running and in an error state");
 480         }
 481         return SUCCESS; # error always succeeds
 482
 483     }
 484
 485     die "implement me (cmd '$cmd')";
 486 }
 487
 488 1;