src/PVE/HA/Sim/Hardware.pm

   1 package PVE::HA::Sim::Hardware;
   2
   3 # Simulate Hardware resources
   4
   5 # power supply for nodes: on/off
   6 # network connection to nodes: on/off
   7 # watchdog devices for nodes
   8
   9 use strict;
  10 use warnings;
  11
  12 use Fcntl qw(:DEFAULT :flock);
  13 use File::Copy;
  14 use File::Path qw(make_path remove_tree);
  15 use IO::File;
  16 use JSON;
  17 use POSIX qw(strftime EINTR);
  18
  19 use PVE::HA::FenceConfig;
  20 use PVE::HA::Groups;
  21
  22 my $watchdog_timeout = 60;
  23
  24 # Status directory layout
  25 #
  26 # configuration
  27 #
  28 # $testdir/cmdlist                    Command list for simulation
  29 # $testdir/hardware_status            Hardware description (number of nodes, ...)
  30 # $testdir/manager_status             CRM status (start with {})
  31 # $testdir/service_config             Service configuration
  32 # $testdir/static_service_stats       Static service usage information (cpu, memory)
  33 # $testdir/groups                     HA groups configuration
  34 # $testdir/service_status_<node>      Service status
  35 # $testdir/datacenter.cfg             Datacenter wide HA configuration
  36
  37 #
  38 # runtime status for simulation system
  39 #
  40 # $testdir/status/cluster_locks        Cluster locks
  41 # $testdir/status/hardware_status      Hardware status (power/network on/off)
  42 # $testdir/status/static_service_stats Static service usage information (cpu, memory)
  43 # $testdir/status/watchdog_status      Watchdog status
  44 #
  45 # runtime status
  46 #
  47 # $testdir/status/lrm_status_<node>           LRM status
  48 # $testdir/status/manager_status              CRM status
  49 # $testdir/status/crm_commands                CRM command queue
  50 # $testdir/status/service_config              Service configuration
  51 # $testdir/status/service_status_<node>       Service status
  52 # $testdir/status/groups                      HA groups configuration
  53
  54 sub read_lrm_status {
  55     my ($self, $node) = @_;
  56
  57     my $filename = "$self->{statusdir}/lrm_status_$node";
  58
  59     return PVE::HA::Tools::read_json_from_file($filename, {});
  60 }
  61
  62 sub write_lrm_status {
  63     my ($self, $node, $status_obj) = @_;
  64
  65     my $filename = "$self->{statusdir}/lrm_status_$node";
  66
  67     PVE::HA::Tools::write_json_to_file($filename, $status_obj);
  68 }
  69
  70 sub read_hardware_status_nolock {
  71     my ($self) = @_;
  72
  73     my $filename = "$self->{statusdir}/hardware_status";
  74
  75     my $raw = PVE::Tools::file_get_contents($filename);
  76     my $cstatus = decode_json($raw);
  77
  78     return $cstatus;
  79 }
  80
  81 sub write_hardware_status_nolock {
  82     my ($self, $cstatus) = @_;
  83
  84     my $filename = "$self->{statusdir}/hardware_status";
  85
  86     PVE::Tools::file_set_contents($filename, encode_json($cstatus));
  87 };
  88
  89 sub read_service_config {
  90     my ($self) = @_;
  91
  92     my $filename = "$self->{statusdir}/service_config";
  93     my $conf = PVE::HA::Tools::read_json_from_file($filename);
  94
  95     foreach my $sid (keys %$conf) {
  96         my $d = $conf->{$sid};
  97
  98         die "service '$sid' without assigned node!" if !$d->{node};
  99
 100         if ($sid =~ m/^(vm|ct|fa):(\d+)$/) {
 101             $d->{type} = $1;
 102             $d->{name} = $2;
 103         } else {
 104             die "implement me";
 105         }
 106         $d->{state} = 'disabled' if !$d->{state};
 107         $d->{state} = 'started' if $d->{state} eq 'enabled'; # backward compatibility
 108         $d->{max_restart} = 1 if !defined($d->{max_restart});
 109         $d->{max_relocate} = 1 if !defined($d->{max_relocate});
 110     }
 111
 112     return $conf;
 113 }
 114
 115 sub update_service_config {
 116     my ($self, $sid, $param) = @_;
 117
 118     my $conf = $self->read_service_config();
 119
 120     my $sconf = $conf->{$sid} || die "no such resource '$sid'\n";
 121
 122     foreach my $k (%$param) {
 123         $sconf->{$k} = $param->{$k};
 124     }
 125
 126     $self->write_service_config($conf);
 127 }
 128
 129 sub write_service_config {
 130     my ($self, $conf) = @_;
 131
 132     $self->{service_config} = $conf;
 133
 134     my $filename = "$self->{statusdir}/service_config";
 135     return PVE::HA::Tools::write_json_to_file($filename, $conf);
 136 }
 137
 138 sub read_fence_config {
 139     my ($self) = @_;
 140
 141     my $raw = undef;
 142
 143     my $filename = "$self->{statusdir}/fence.cfg";
 144     if (-e $filename) {
 145         $raw = PVE::Tools::file_get_contents($filename);
 146     }
 147
 148     return PVE::HA::FenceConfig::parse_config($filename, $raw);
 149 }
 150
 151 sub exec_fence_agent {
 152     my ($self, $agent, $node, @param) = @_;
 153
 154     # let all agent succeed and behave the same for now
 155     $self->sim_hardware_cmd("power $node off", $agent);
 156
 157     return 0; # EXIT_SUCCESS
 158 }
 159
 160 sub set_service_state {
 161     my ($self, $sid, $state) = @_;
 162
 163     my $conf = $self->read_service_config();
 164     die "no such service '$sid'" if !$conf->{$sid};
 165
 166     $conf->{$sid}->{state} = $state;
 167
 168     $self->write_service_config($conf);
 169
 170     return $conf;
 171 }
 172
 173 sub add_service {
 174     my ($self, $sid, $opts) = @_;
 175
 176     my $conf = $self->read_service_config();
 177     die "resource ID '$sid' already defined\n" if $conf->{$sid};
 178
 179     $conf->{$sid} = $opts;
 180     $conf->{$sid}->@{qw(type name)} = split(/:/, $sid);
 181
 182     $self->write_service_config($conf);
 183
 184     return $conf;
 185 }
 186
 187 sub delete_service {
 188     my ($self, $sid) = @_;
 189
 190     my $conf = $self->read_service_config();
 191
 192     die "no such service '$sid'" if !$conf->{$sid};
 193
 194     delete $conf->{$sid};
 195
 196     $self->write_service_config($conf);
 197
 198     return $conf;
 199 }
 200
 201 sub change_service_location {
 202     my ($self, $sid, $current_node, $new_node) = @_;
 203
 204     my $conf = $self->read_service_config();
 205
 206     die "no such service '$sid'\n" if !$conf->{$sid};
 207
 208     die "current_node for '$sid' does not match ($current_node != $conf->{$sid}->{node})\n"
 209         if $current_node ne $conf->{$sid}->{node};
 210
 211     $conf->{$sid}->{node} = $new_node;
 212
 213     $self->write_service_config($conf);
 214 }
 215
 216 sub service_has_lock {
 217     my ($self, $sid) = @_;
 218
 219     my $conf = $self->read_service_config();
 220
 221     die "no such service '$sid'\n" if !$conf->{$sid};
 222
 223     return $conf->{$sid}->{lock};
 224 }
 225
 226 sub lock_service {
 227     my ($self, $sid, $lock) = @_;
 228
 229     my $conf = $self->read_service_config();
 230
 231     die "no such service '$sid'\n" if !$conf->{$sid};
 232
 233     $conf->{$sid}->{lock} = $lock || 'backup';
 234
 235     $self->write_service_config($conf);
 236
 237     return $conf;
 238 }
 239
 240 sub unlock_service {
 241     my ($self, $sid, $lock) = @_;
 242
 243     my $conf = $self->read_service_config();
 244
 245     die "no such service '$sid'\n" if !$conf->{$sid};
 246
 247     if (!defined($conf->{$sid}->{lock})) {
 248         return undef;
 249     }
 250
 251     if (defined($lock) && $conf->{$sid}->{lock} ne $lock) {
 252         warn "found lock '$conf->{$sid}->{lock}' trying to remove '$lock' lock\n";
 253         return undef;
 254     }
 255
 256     my $removed_lock = delete $conf->{$sid}->{lock};
 257
 258     $self->write_service_config($conf);
 259
 260     return $removed_lock;
 261 }
 262
 263 sub queue_crm_commands_nolock {
 264     my ($self, $cmd) = @_;
 265
 266     chomp $cmd;
 267
 268     my $data = '';
 269     my $filename = "$self->{statusdir}/crm_commands";
 270     if (-f $filename) {
 271         $data = PVE::Tools::file_get_contents($filename);
 272     }
 273     $data .= "$cmd\n";
 274     PVE::Tools::file_set_contents($filename, $data);
 275
 276     return undef;
 277 }
 278
 279 sub queue_crm_commands {
 280     my ($self, $cmd) = @_;
 281
 282     my $code = sub { $self->queue_crm_commands_nolock($cmd); };
 283
 284     $self->global_lock($code);
 285
 286     return undef;
 287 }
 288
 289 sub read_crm_commands {
 290     my ($self) = @_;
 291
 292     my $code = sub {
 293         my $data = '';
 294
 295         my $filename = "$self->{statusdir}/crm_commands";
 296         if (-f $filename) {
 297             $data = PVE::Tools::file_get_contents($filename);
 298         }
 299         PVE::Tools::file_set_contents($filename, '');
 300
 301         return $data;
 302     };
 303
 304     return $self->global_lock($code);
 305 }
 306
 307 sub read_group_config {
 308     my ($self) = @_;
 309
 310     my $filename = "$self->{statusdir}/groups";
 311     my $raw = '';
 312     $raw = PVE::Tools::file_get_contents($filename) if -f $filename;
 313
 314     return PVE::HA::Groups->parse_config($filename, $raw);
 315 }
 316
 317 sub read_service_status {
 318     my ($self, $node) = @_;
 319
 320     my $filename = "$self->{statusdir}/service_status_$node";
 321     return PVE::HA::Tools::read_json_from_file($filename);
 322 }
 323
 324 sub write_service_status {
 325     my ($self, $node, $data) = @_;
 326
 327     my $filename = "$self->{statusdir}/service_status_$node";
 328     my $res = PVE::HA::Tools::write_json_to_file($filename, $data);
 329
 330     # fixme: add test if a service runs on two nodes!!!
 331
 332     return $res;
 333 }
 334
 335 sub read_static_service_stats {
 336     my ($self) = @_;
 337
 338     my $filename = "$self->{statusdir}/static_service_stats";
 339     my $stats = eval { PVE::HA::Tools::read_json_from_file($filename) };
 340     $self->log('error', "loading static service stats failed - $@") if $@;
 341
 342     return $stats;
 343 }
 344
 345 my $default_group_config = <<__EOD;
 346 group: prefer_node1
 347     nodes node1
 348     nofailback 1
 349
 350 group: prefer_node2
 351     nodes node2
 352     nofailback 1
 353
 354 group: prefer_node3
 355     nodes node3
 356     nofailback 1
 357 __EOD
 358
 359 sub new {
 360     my ($this, $testdir) = @_;
 361
 362     die "missing testdir" if !$testdir;
 363
 364     die "testdir '$testdir' does not exist or is not a directory!\n"
 365         if !-d $testdir;
 366
 367     my $class = ref($this) || $this;
 368
 369     my $self = bless {}, $class;
 370
 371     my $statusdir = $self->{statusdir} = "$testdir/status";
 372
 373     remove_tree($statusdir);
 374     mkdir $statusdir;
 375
 376     # copy initial configuartion
 377     copy("$testdir/manager_status", "$statusdir/manager_status"); # optional
 378
 379     if (-f "$testdir/groups") {
 380         copy("$testdir/groups", "$statusdir/groups");
 381     } else {
 382         PVE::Tools::file_set_contents("$statusdir/groups", $default_group_config);
 383     }
 384
 385     if (-f "$testdir/service_config") {
 386         copy("$testdir/service_config", "$statusdir/service_config");
 387     } else {
 388         my $conf = {
 389             'vm:101' => { node => 'node1', group => 'prefer_node1' },
 390             'vm:102' => { node => 'node2', group => 'prefer_node2' },
 391             'vm:103' => { node => 'node3', group => 'prefer_node3' },
 392             'vm:104' => { node => 'node1', group => 'prefer_node1' },
 393             'vm:105' => { node => 'node2', group => 'prefer_node2' },
 394             'vm:106' => { node => 'node3', group => 'prefer_node3' },
 395         };
 396         $self->write_service_config($conf);
 397     }
 398
 399     if (-f "$testdir/hardware_status") {
 400         copy("$testdir/hardware_status", "$statusdir/hardware_status") ||
 401             die "Copy failed: $!\n";
 402     } else {
 403         my $cstatus = {
 404             node1 => { power => 'off', network => 'off' },
 405             node2 => { power => 'off', network => 'off' },
 406             node3 => { power => 'off', network => 'off' },
 407         };
 408         $self->write_hardware_status_nolock($cstatus);
 409     }
 410
 411     if (-f "$testdir/fence.cfg") {
 412         copy("$testdir/fence.cfg", "$statusdir/fence.cfg");
 413     }
 414
 415     if (-f "$testdir/datacenter.cfg") {
 416         copy("$testdir/datacenter.cfg", "$statusdir/datacenter.cfg");
 417     }
 418
 419     if (-f "$testdir/static_service_stats") {
 420         copy("$testdir/static_service_stats", "$statusdir/static_service_stats");
 421     }
 422
 423     my $cstatus = $self->read_hardware_status_nolock();
 424
 425     foreach my $node (sort keys %$cstatus) {
 426         $self->{nodes}->{$node} = {};
 427
 428         if (-f "$testdir/service_status_$node") {
 429             copy("$testdir/service_status_$node", "$statusdir/service_status_$node");
 430         } else {
 431             $self->write_service_status($node, {});
 432         }
 433     }
 434
 435     $self->{service_config} = $self->read_service_config();
 436
 437     return $self;
 438 }
 439
 440 sub get_time {
 441     my ($self) = @_;
 442
 443     die "implement in subclass";
 444 }
 445
 446 sub log {
 447     my ($self, $level, $msg, $id) = @_;
 448
 449     chomp $msg;
 450
 451     my $time = $self->get_time();
 452
 453     $id = 'hardware' if !$id;
 454
 455     printf("%-5s %5d %12s: $msg\n", $level, $time, $id);
 456 }
 457
 458 sub statusdir {
 459     my ($self, $node) = @_;
 460
 461     return $self->{statusdir};
 462 }
 463
 464 sub read_datacenter_conf {
 465     my ($self, $node) = @_;
 466
 467     my $filename = "$self->{statusdir}/datacenter.cfg";
 468     return PVE::HA::Tools::read_json_from_file($filename, {});
 469 }
 470
 471 sub global_lock {
 472     my ($self, $code, @param) = @_;
 473
 474     my $lockfile = "$self->{statusdir}/hardware.lck";
 475     my $fh = IO::File->new(">>$lockfile") ||
 476         die "unable to open '$lockfile'\n";
 477
 478     my $success;
 479     for (;;) {
 480         $success = flock($fh, LOCK_EX);
 481         if ($success || ($! != EINTR)) {
 482             last;
 483         }
 484         if (!$success) {
 485             close($fh);
 486             die "can't acquire lock '$lockfile' - $!\n";
 487         }
 488     }
 489
 490     my $res;
 491
 492     eval { $res = &$code($fh, @param) };
 493     my $err = $@;
 494
 495     close($fh);
 496
 497     die $err if $err;
 498
 499     return $res;
 500 }
 501
 502 my $compute_node_info = sub {
 503     my ($self, $cstatus) = @_;
 504
 505     my $node_info = {};
 506
 507     my $node_count = 0;
 508     my $online_count = 0;
 509
 510     foreach my $node (keys %$cstatus) {
 511         my $d = $cstatus->{$node};
 512
 513         my $online = ($d->{power} eq 'on' && $d->{network} eq 'on') ? 1 : 0;
 514         $node_info->{$node}->{online} = $online;
 515
 516         $node_count++;
 517         $online_count++ if $online;
 518     }
 519
 520     my $quorate = ($online_count > int($node_count/2)) ? 1 : 0;
 521
 522     if (!$quorate) {
 523         foreach my $node (keys %$cstatus) {
 524             my $d = $cstatus->{$node};
 525             $node_info->{$node}->{online} = 0;
 526         }
 527     }
 528
 529     return ($node_info, $quorate);
 530 };
 531
 532 sub get_node_info {
 533     my ($self) = @_;
 534
 535     my $cstatus = $self->read_hardware_status_nolock();
 536     my ($node_info, $quorate) = &$compute_node_info($self, $cstatus);
 537
 538     return ($node_info, $quorate);
 539 }
 540
 541 # helper for Sim/ only
 542 sub get_cfs_state {
 543     my ($self, $node, $state) = @_;
 544
 545     # TODO: ensure nolock is OK when adding this to RTSim
 546     my $cstatus = $self->read_hardware_status_nolock();
 547     my $res = $cstatus->{$node}->{cfs}->{$state};
 548
 549     # we assume default true if not defined
 550     return !defined($res) || $res;
 551 }
 552
 553 # simulate hardware commands, the following commands are available:
 554 #   power <node> <on|off>
 555 #   network <node> <on|off>
 556 #   delay <seconds>
 557 #   skip-round <crm|lrm> [<rounds=1>]
 558 #   cfs <node> <rw|update> <work|fail>
 559 #   reboot <node>
 560 #   shutdown <node>
 561 #   restart-lrm <node>
 562 #   service <sid> <started|disabled|stopped|ignored>
 563 #   service <sid> <migrate|relocate> <target>
 564 #   service <sid> stop <timeout>
 565 #   service <sid> lock/unlock [lockname]
 566 #   service <sid> <add|delete>
 567 sub sim_hardware_cmd {
 568     my ($self, $cmdstr, $logid) = @_;
 569
 570     my $code = sub {
 571         my ($lock_fh) = @_;
 572
 573         my $cstatus = $self->read_hardware_status_nolock();
 574
 575         my ($cmd, $objid, $action, @params) = split(/\s+/, $cmdstr);
 576         my $param = $params[0]; # for convenience/legacy
 577
 578         die "sim_hardware_cmd: no node or service for command specified"
 579             if !$objid;
 580
 581         my ($node, $sid, $d);
 582
 583         if ($cmd eq 'service') {
 584             $sid = PVE::HA::Tools::pve_verify_ha_resource_id($objid);
 585         } else {
 586             $node = $objid;
 587             $d = $self->{nodes}->{$node} ||
 588                 die "sim_hardware_cmd: no such node '$node'\n";
 589         }
 590
 591         $self->log('info', "execute $cmdstr", $logid);
 592
 593         if ($cmd eq 'power') {
 594             die "sim_hardware_cmd: unknown action '$action'\n"
 595                 if $action !~ m/^(on|off)$/;
 596
 597             if ($cstatus->{$node}->{power} ne $action) {
 598                 if ($action eq 'on') {
 599
 600                     $d->{crm} = $self->crm_control('start', $d, $lock_fh) if !defined($d->{crm});
 601                     $d->{lrm} = $self->lrm_control('start', $d, $lock_fh) if !defined($d->{lrm});
 602                     $d->{lrm_restart} = undef;
 603                     $cstatus->{$node}->{cfs} = {};
 604
 605                 } else {
 606
 607                     if ($d->{crm}) {
 608                         $d->{crm_env}->log('info', "killed by poweroff");
 609                         $self->crm_control('stop', $d, $lock_fh);
 610                         $d->{crm} = undef;
 611                     }
 612                     if ($d->{lrm}) {
 613                         $d->{lrm_env}->log('info', "killed by poweroff");
 614                         $self->lrm_control('stop', $d, $lock_fh);
 615                         $d->{lrm} = undef;
 616                         $d->{lrm_restart} = undef;
 617                     }
 618
 619                     $self->watchdog_reset_nolock($node);
 620                     $self->write_service_status($node, {});
 621                 }
 622             }
 623
 624             $cstatus->{$node}->{power} = $action;
 625             $cstatus->{$node}->{network} = $action;
 626             $cstatus->{$node}->{shutdown} = undef;
 627
 628             $self->write_hardware_status_nolock($cstatus);
 629
 630         } elsif ($cmd eq 'network') {
 631             die "sim_hardware_cmd: unknown network action '$action'"
 632                 if $action !~ m/^(on|off)$/;
 633             $cstatus->{$node}->{network} = $action;
 634
 635             $self->write_hardware_status_nolock($cstatus);
 636
 637         } elsif ($cmd eq 'cfs') {
 638             die "sim_hardware_cmd: unknown cfs action '$action' for node '$node'"
 639                 if $action !~ m/^(rw|update)$/;
 640             die "sim_hardware_cmd: unknown cfs command '$param' for '$action' on node '$node'"
 641                 if $param !~ m/^(work|fail)$/;
 642
 643             $cstatus->{$node}->{cfs}->{$action} = $param eq 'work';
 644             $self->write_hardware_status_nolock($cstatus);
 645
 646         } elsif ($cmd eq 'reboot' || $cmd eq 'shutdown') {
 647             $cstatus->{$node}->{shutdown} = $cmd;
 648
 649             $self->write_hardware_status_nolock($cstatus);
 650
 651             $self->lrm_control('shutdown', $d, $lock_fh) if defined($d->{lrm});
 652         } elsif ($cmd eq 'restart-lrm') {
 653             if ($d->{lrm}) {
 654                 $d->{lrm_restart} = 1;
 655                 $self->lrm_control('shutdown', $d, $lock_fh);
 656             }
 657         } elsif ($cmd eq 'crm') {
 658
 659             if ($action eq 'stop') {
 660                 if ($d->{crm}) {
 661                     $d->{crm_stop} = 1;
 662                     $self->crm_control('shutdown', $d, $lock_fh);
 663                 }
 664             } elsif ($action eq 'start') {
 665                 $d->{crm} = $self->crm_control('start', $d, $lock_fh) if !defined($d->{crm});
 666             } elsif ($action eq 'enable-node-maintenance' || $action eq 'disable-node-maintenance') {
 667                 $self->queue_crm_commands_nolock("$action $node");
 668             } else {
 669                 die "sim_hardware_cmd: unknown action '$action'";
 670             }
 671
 672         } elsif ($cmd eq 'service') {
 673             if ($action eq 'started' || $action eq 'disabled' ||
 674                 $action eq 'stopped' || $action eq 'ignored') {
 675
 676                 $self->set_service_state($sid, $action);
 677
 678             } elsif ($action eq 'migrate' || $action eq 'relocate') {
 679
 680                 die "sim_hardware_cmd: missing target node for '$action' command"
 681                     if !$param;
 682
 683                 $self->queue_crm_commands_nolock("$action $sid $param");
 684
 685             } elsif ($action eq 'stop') {
 686
 687                 die "sim_hardware_cmd: missing timeout for '$action' command"
 688                     if !defined($param);
 689
 690                 $self->queue_crm_commands_nolock("$action $sid $param");
 691
 692             } elsif ($action eq 'add') {
 693
 694                 $self->add_service($sid, {state => $params[1] || 'started', node => $param});
 695
 696             } elsif ($action eq 'delete') {
 697
 698                 $self->delete_service($sid);
 699
 700             } elsif ($action eq 'lock') {
 701
 702                 $self->lock_service($sid, $param);
 703
 704             } elsif ($action eq 'unlock') {
 705
 706                 $self->unlock_service($sid, $param);
 707
 708             } else {
 709                 die "sim_hardware_cmd: unknown service action '$action' " .
 710                     "- not implemented\n"
 711             }
 712         } else {
 713             die "sim_hardware_cmd: unknown command '$cmdstr'\n";
 714         }
 715
 716         return $cstatus;
 717     };
 718
 719     return $self->global_lock($code);
 720 }
 721
 722 # for controlling the resource manager services
 723 sub crm_control {
 724     my ($self, $action, $data, $lock_fh) = @_;
 725
 726     die "implement in subclass";
 727 }
 728
 729 sub lrm_control {
 730     my ($self, $action, $data, $lock_fh) = @_;
 731
 732     die "implement in subclass";
 733 }
 734
 735 sub run {
 736     my ($self) = @_;
 737
 738     die "implement in subclass";
 739 }
 740
 741 my $modify_watchog = sub {
 742     my ($self, $code) = @_;
 743
 744     my $update_cmd = sub {
 745
 746         my $filename = "$self->{statusdir}/watchdog_status";
 747
 748         my ($res, $wdstatus);
 749
 750         if (-f $filename) {
 751             my $raw = PVE::Tools::file_get_contents($filename);
 752             $wdstatus = decode_json($raw);
 753         } else {
 754             $wdstatus = {};
 755         }
 756
 757         ($wdstatus, $res) = &$code($wdstatus);
 758
 759         PVE::Tools::file_set_contents($filename, encode_json($wdstatus));
 760
 761         return $res;
 762     };
 763
 764     return $self->global_lock($update_cmd);
 765 };
 766
 767 sub watchdog_reset_nolock {
 768     my ($self, $node) = @_;
 769
 770     my $filename = "$self->{statusdir}/watchdog_status";
 771
 772     if (-f $filename) {
 773         my $raw = PVE::Tools::file_get_contents($filename);
 774         my $wdstatus = decode_json($raw);
 775
 776         foreach my $id (keys %$wdstatus) {
 777             delete $wdstatus->{$id} if $wdstatus->{$id}->{node} eq $node;
 778         }
 779
 780         PVE::Tools::file_set_contents($filename, encode_json($wdstatus));
 781     }
 782 }
 783
 784 sub watchdog_check {
 785     my ($self, $node) = @_;
 786
 787     my $code = sub {
 788         my ($wdstatus) = @_;
 789
 790         my $res = 1;
 791
 792         foreach my $wfh (keys %$wdstatus) {
 793             my $wd = $wdstatus->{$wfh};
 794             next if $wd->{node} ne $node;
 795
 796             my $ctime = $self->get_time();
 797             my $tdiff = $ctime - $wd->{update_time};
 798
 799             if ($tdiff > $watchdog_timeout) { # expired
 800                 $res = 0;
 801                 delete $wdstatus->{$wfh};
 802             }
 803         }
 804
 805         return ($wdstatus, $res);
 806     };
 807
 808     return &$modify_watchog($self, $code);
 809 }
 810
 811 my $wdcounter = 0;
 812
 813 sub watchdog_open {
 814     my ($self, $node) = @_;
 815
 816     my $code = sub {
 817         my ($wdstatus) = @_;
 818
 819         ++$wdcounter;
 820
 821         my $id = "WD:$node:$$:$wdcounter";
 822
 823         die "internal error" if defined($wdstatus->{$id});
 824
 825         $wdstatus->{$id} = {
 826             node => $node,
 827             update_time => $self->get_time(),
 828         };
 829
 830         return ($wdstatus, $id);
 831     };
 832
 833     return &$modify_watchog($self, $code);
 834 }
 835
 836 sub watchdog_close {
 837     my ($self, $wfh) = @_;
 838
 839     my $code = sub {
 840         my ($wdstatus) = @_;
 841
 842         my $wd = $wdstatus->{$wfh};
 843         die "no such watchdog handle '$wfh'\n" if !defined($wd);
 844
 845         my $tdiff = $self->get_time() - $wd->{update_time};
 846         die "watchdog expired" if $tdiff > $watchdog_timeout;
 847
 848         delete $wdstatus->{$wfh};
 849
 850         return ($wdstatus);
 851     };
 852
 853     return &$modify_watchog($self, $code);
 854 }
 855
 856 sub watchdog_update {
 857     my ($self, $wfh) = @_;
 858
 859     my $code = sub {
 860         my ($wdstatus) = @_;
 861
 862         my $wd = $wdstatus->{$wfh};
 863
 864         die "no such watchdog handle '$wfh'\n" if !defined($wd);
 865
 866         my $ctime = $self->get_time();
 867         my $tdiff = $ctime - $wd->{update_time};
 868
 869         die "watchdog expired" if $tdiff > $watchdog_timeout;
 870
 871         $wd->{update_time} = $ctime;
 872
 873         return ($wdstatus);
 874     };
 875
 876     return &$modify_watchog($self, $code);
 877 }
 878
 879 sub get_static_node_stats {
 880     my ($self) = @_;
 881
 882     my $cstatus = $self->read_hardware_status_nolock();
 883
 884     my $stats = {};
 885     for my $node (keys $cstatus->%*) {
 886         $stats->{$node} = { $cstatus->{$node}->%{qw(cpus memory)} };
 887     }
 888
 889     return $stats;
 890 }
 891
 892 1;