src/PVE/HA/Sim/RTHardware.pm

   1 package PVE::HA::Sim::RTHardware;
   2
   3 # Simulate Hardware resources in Realtime by
   4 # running CRM and LRM in separate processes
   5
   6 use strict;
   7 use warnings;
   8 use POSIX qw(strftime EINTR);
   9 use Data::Dumper;
  10 $Data::Dumper::Sortkeys = 1; # fixes 'random' output behaviour of manager status
  11 use JSON;
  12 use IO::File;
  13 use IO::Select;
  14 use Fcntl qw(:DEFAULT :flock);
  15 use File::Copy;
  16 use File::Path qw(make_path remove_tree);
  17
  18 use Glib;
  19
  20 use Gtk3 '-init';
  21
  22 use PVE::HA::CRM;
  23 use PVE::HA::LRM;
  24
  25 use PVE::HA::Sim::RTEnv;
  26 use base qw(PVE::HA::Sim::Hardware);
  27
  28 sub new {
  29     my ($this, $testdir) = @_;
  30
  31     my $class = ref($this) || $this;
  32
  33     my $self = $class->SUPER::new($testdir);
  34
  35     my $logfile = "$testdir/log";
  36     $self->{logfh} = IO::File->new(">$logfile") ||
  37         die "unable to open '$logfile' - $!";
  38
  39     foreach my $node (sort keys %{$self->{nodes}}) {
  40         my $d = $self->{nodes}->{$node};
  41
  42         $d->{crm} = undef; # create on power on
  43         $d->{lrm} = undef; # create on power on
  44     }
  45
  46     $self->create_main_window();
  47
  48     return $self;
  49 }
  50
  51 sub get_time {
  52     my ($self) = @_;
  53
  54     return time();
  55 }
  56
  57 sub log {
  58     my ($self, $level, $msg, $id) = @_;
  59
  60     chomp $msg;
  61
  62     my $time = $self->get_time();
  63
  64     $id = 'hardware' if !$id;
  65
  66     my $text = sprintf("%-5s %10s %12s: $msg\n", $level,
  67                        strftime("%H:%M:%S", localtime($time)), $id);
  68
  69     $self->append_text($text);
  70 }
  71
  72 # fixme: duplicate code in Env?
  73 sub read_manager_status {
  74     my ($self) = @_;
  75
  76     my $filename = "$self->{statusdir}/manager_status";
  77
  78     return PVE::HA::Tools::read_json_from_file($filename, {});
  79 }
  80
  81 sub fork_daemon {
  82     my ($self, $lockfh, $type, $node) = @_;
  83
  84     my @psync = POSIX::pipe();
  85
  86     my $pid = fork();
  87     die "fork failed" if ! defined($pid);
  88
  89     if ($pid == 0) {
  90
  91         close($lockfh) if defined($lockfh); # unlock global lock
  92
  93         POSIX::close($psync[0]);
  94
  95         my $outfh = $psync[1];
  96
  97         my $fd = fileno (STDIN);
  98         close STDIN;
  99         POSIX::close(0) if $fd != 0;
 100
 101         die "unable to redirect STDIN - $!"
 102             if !open(STDIN, "</dev/null");
 103
 104         # redirect STDOUT
 105         $fd = fileno(STDOUT);
 106         close STDOUT;
 107         POSIX::close (1) if $fd != 1;
 108
 109         die "unable to redirect STDOUT - $!"
 110             if !open(STDOUT, ">&", $outfh);
 111
 112         STDOUT->autoflush (1);
 113
 114         #  redirect STDERR to STDOUT
 115         $fd = fileno(STDERR);
 116         close STDERR;
 117         POSIX::close(2) if $fd != 2;
 118
 119         die "unable to redirect STDERR - $!"
 120             if !open(STDERR, ">&1");
 121
 122         STDERR->autoflush(1);
 123
 124         if ($type eq 'crm') {
 125
 126             my $haenv = PVE::HA::Env->new('PVE::HA::Sim::RTEnv', $node, $self, 'crm');
 127
 128             my $crm = PVE::HA::CRM->new($haenv);
 129
 130             for (;;) {
 131                 $haenv->loop_start_hook();
 132
 133                 if (!$crm->do_one_iteration()) {
 134                     $haenv->log("info", "daemon stopped");
 135                     exit (0);
 136                 }
 137
 138                 $haenv->loop_end_hook();
 139             }
 140
 141         } else {
 142
 143             my $haenv = PVE::HA::Env->new('PVE::HA::Sim::RTEnv', $node, $self, 'lrm');
 144
 145             my $lrm = PVE::HA::LRM->new($haenv);
 146
 147             for (;;) {
 148                 $haenv->loop_start_hook();
 149
 150                 if (!$lrm->do_one_iteration()) {
 151                     $haenv->log("info", "daemon stopped");
 152                     exit (0);
 153                 }
 154
 155                 $haenv->loop_end_hook();
 156             }
 157         }
 158
 159         exit(-1);
 160     }
 161
 162     # parent
 163
 164     POSIX::close ($psync[1]);
 165
 166     Glib::IO->add_watch($psync[0], ['in', 'hup'], sub {
 167         my ($fd, $cond) = @_;
 168         if ($cond eq 'in') {
 169             my $readbuf;
 170             if (my $count = POSIX::read($fd, $readbuf, 8192)) {
 171                 $self->append_text($readbuf);
 172             }
 173             return 1;
 174         } else {
 175             POSIX::close($fd);
 176             return 0;
 177         }
 178     });
 179
 180     return $pid;
 181 }
 182
 183 # simulate hardware commands
 184 # power <node> <on|off>
 185 # network <node> <on|off>
 186
 187 sub sim_hardware_cmd {
 188     my ($self, $cmdstr, $logid) = @_;
 189
 190     my $cstatus;
 191
 192     # note: do not fork when we own the lock!
 193     my $code = sub {
 194         my ($lockfh) = @_;
 195
 196         $cstatus = $self->read_hardware_status_nolock();
 197
 198         my ($cmd, $node, $action) = split(/\s+/, $cmdstr);
 199
 200         die "sim_hardware_cmd: no node specified" if !$node;
 201         die "sim_hardware_cmd: unknown action '$action'" if $action !~ m/^(on|off)$/;
 202
 203         my $d = $self->{nodes}->{$node};
 204         die "sim_hardware_cmd: no such node '$node'\n" if !$d;
 205
 206         $self->log('info', "execute $cmdstr", $logid);
 207
 208         if ($cmd eq 'power') {
 209             if ($cstatus->{$node}->{power} ne $action) {
 210                 if ($action eq 'on') {
 211                     $d->{crm} = $self->fork_daemon($lockfh, 'crm', $node) if !$d->{crm};
 212                     $d->{lrm} = $self->fork_daemon($lockfh, 'lrm', $node) if !$d->{lrm};
 213                 } else {
 214                     if ($d->{crm}) {
 215                         $self->log('info', "crm on node '$node' killed by poweroff");
 216                         kill(9, $d->{crm});
 217                         $d->{crm} = undef;
 218                     }
 219                     if ($d->{lrm}) {
 220                         $self->log('info', "lrm on node '$node' killed by poweroff");
 221                         kill(9, $d->{lrm});
 222                         $d->{lrm} = undef;
 223                     }
 224                     $self->watchdog_reset_nolock($node);
 225                     $self->write_service_status($node, {});
 226                 }
 227             }
 228
 229             $cstatus->{$node}->{power} = $action;
 230             $cstatus->{$node}->{network} = $action;
 231
 232         } elsif ($cmd eq 'network') {
 233             $cstatus->{$node}->{network} = $action;
 234         } else {
 235             die "sim_hardware_cmd: unknown command '$cmd'\n";
 236         }
 237
 238         $self->write_hardware_status_nolock($cstatus);
 239     };
 240
 241     my $res = $self->global_lock($code);
 242
 243     # update GUI outside lock
 244
 245     foreach my $node (keys %$cstatus) {
 246         my $d = $self->{nodes}->{$node};
 247         $d->{network_btn}->set_active($cstatus->{$node}->{network} eq 'on');
 248         $d->{power_btn}->set_active($cstatus->{$node}->{power} eq 'on');
 249     }
 250
 251     return $res;
 252 }
 253
 254 sub cleanup {
 255     my ($self) = @_;
 256
 257     my @nodes = sort keys %{$self->{nodes}};
 258     foreach my $node (@nodes) {
 259         my $d = $self->{nodes}->{$node};
 260
 261         if ($d->{crm}) {
 262             kill 9, $d->{crm};
 263             delete $d->{crm};
 264         }
 265         if ($d->{lrm}) {
 266             kill 9, $d->{lrm};
 267             delete $d->{lrm};
 268         }
 269     }
 270 }
 271
 272 sub append_text {
 273     my ($self, $text) = @_;
 274
 275     $self->{logfh}->print($text);
 276     $self->{logfh}->flush();
 277
 278     my $logview = $self->{gui}->{text_view} || die "GUI not ready";
 279     my $textbuf = $logview->get_buffer();
 280
 281     $textbuf->insert_at_cursor($text, -1);
 282     my $lines = $textbuf->get_line_count();
 283
 284     my $history = 102;
 285
 286     if ($lines > $history) {
 287         my $start = $textbuf->get_iter_at_line(0);
 288         my $end =  $textbuf->get_iter_at_line($lines - $history);
 289         $textbuf->delete($start, $end);
 290     }
 291
 292     $logview->scroll_to_mark($textbuf->get_insert(), 0.0, 1, 0.0, 1.0);
 293 }
 294
 295 sub set_power_state {
 296     my ($self, $node) = @_;
 297
 298     my $d = $self->{nodes}->{$node} || die "no such node '$node'";
 299
 300     my $action = $d->{power_btn}->get_active() ? 'on' : 'off';
 301
 302     $self->sim_hardware_cmd("power $node $action");
 303 }
 304
 305 sub set_network_state {
 306     my ($self, $node) = @_;
 307
 308     my $d = $self->{nodes}->{$node} || die "no such node '$node'";
 309
 310     my $action = $d->{network_btn}->get_active() ? 'on' : 'off';
 311
 312     $self->sim_hardware_cmd("network $node $action");
 313 }
 314
 315 sub set_service_state {
 316     my ($self, $sid) = @_;
 317
 318     my $d = $self->{service_gui}->{$sid} || die "no such service '$sid'";
 319     my $state = $d->{enable_btn}->get_active() ? 'enabled' : 'disabled';
 320
 321     $self->{service_config} = $self->SUPER::set_service_state($sid, $state);
 322
 323 }
 324
 325 sub create_node_control {
 326     my ($self) = @_;
 327
 328     my $ngrid = Gtk3::Grid->new();
 329     $ngrid->set_row_spacing(2);
 330     $ngrid->set_column_spacing(5);
 331     $ngrid->set('margin-left', 5);
 332
 333     my $w = Gtk3::Label->new('Node');
 334     $ngrid->attach($w, 0, 0, 1, 1);
 335     $w = Gtk3::Label->new('Power');
 336     $ngrid->attach($w, 1, 0, 1, 1);
 337     $w = Gtk3::Label->new('Network');
 338     $ngrid->attach($w, 2, 0, 1, 1);
 339     $w = Gtk3::Label->new('Status');
 340     $w->set_size_request(150, -1);
 341     $w->set_alignment (0, 0.5);
 342     $ngrid->attach($w, 3, 0, 1, 1);
 343
 344     my $row = 1;
 345
 346     my @nodes = sort keys %{$self->{nodes}};
 347
 348     foreach my $node (@nodes) {
 349         my $d = $self->{nodes}->{$node};
 350
 351         $w = Gtk3::Label->new($node);
 352         $ngrid->attach($w, 0, $row, 1, 1);
 353         $w = Gtk3::Switch->new();
 354         $ngrid->attach($w, 1, $row, 1, 1);
 355         $d->{power_btn} = $w;
 356         $w->signal_connect('notify::active' => sub {
 357             $self->set_power_state($node);
 358         }),
 359
 360         $w = Gtk3::Switch->new();
 361         $ngrid->attach($w, 2, $row, 1, 1);
 362         $d->{network_btn} = $w;
 363         $w->signal_connect('notify::active' => sub {
 364             $self->set_network_state($node);
 365         }),
 366
 367         $w = Gtk3::Label->new('-');
 368         $w->set_alignment (0, 0.5);
 369         $ngrid->attach($w, 3, $row, 1, 1);
 370         $d->{node_status_label} = $w;
 371
 372         $row++;
 373     }
 374
 375     return $ngrid;
 376 }
 377
 378 sub show_migrate_dialog {
 379     my ($self, $sid) = @_;
 380
 381     my $dialog = Gtk3::Dialog->new();
 382
 383     $dialog->set_title("Migrate $sid");
 384     $dialog->set_modal(1);
 385
 386     my $grid = Gtk3::Grid->new();
 387     $grid->set_row_spacing(2);
 388     $grid->set_column_spacing(5);
 389     $grid->set('margin', 5);
 390
 391     my $w = Gtk3::Label->new('Target Mode');
 392     $grid->attach($w, 0, 0, 1, 1);
 393
 394     my @nodes = sort keys %{$self->{nodes}};
 395     $w = Gtk3::ComboBoxText->new();
 396     foreach my $node (@nodes) {
 397         $w->append_text($node);
 398     }
 399
 400     my $target = '';
 401     $w->signal_connect('notify::active' => sub {
 402         my $w = shift;
 403
 404         my $sel = $w->get_active();
 405         return if $sel < 0;
 406
 407         $target = $nodes[$sel];
 408     });
 409     $grid->attach($w, 1, 0, 1, 1);
 410
 411     my $relocate_btn = Gtk3::CheckButton->new_with_label("stop service (relocate)");
 412     $grid->attach($relocate_btn, 1, 1, 1, 1);
 413
 414     my $contarea = $dialog->get_content_area();
 415
 416     $contarea->add($grid);
 417
 418     $dialog->add_button("_OK", 1);
 419
 420     $dialog->show_all();
 421     my $res = $dialog->run();
 422
 423     $dialog->destroy();
 424
 425     if ($res == 1 && $target) {
 426         if ($relocate_btn->get_active()) {
 427             $self->queue_crm_commands("relocate $sid $target");
 428         } else {
 429             $self->queue_crm_commands("migrate $sid $target");
 430         }
 431     }
 432 }
 433
 434 sub create_service_control {
 435     my ($self) = @_;
 436
 437     my $sgrid = Gtk3::Grid->new();
 438     $sgrid->set_row_spacing(2);
 439     $sgrid->set_column_spacing(5);
 440     $sgrid->set('margin', 5);
 441
 442     my $w = Gtk3::Label->new('Service');
 443     $sgrid->attach($w, 0, 0, 1, 1);
 444     $w = Gtk3::Label->new('Enable');
 445     $sgrid->attach($w, 1, 0, 1, 1);
 446     $w = Gtk3::Label->new('Node');
 447     $sgrid->attach($w, 3, 0, 1, 1);
 448     $w = Gtk3::Label->new('Status');
 449     $w->set_alignment (0, 0.5);
 450     $w->set_size_request(150, -1);
 451     $sgrid->attach($w, 4, 0, 1, 1);
 452
 453     my $row = 1;
 454     my @nodes = keys %{$self->{nodes}};
 455
 456     foreach my $sid (sort keys %{$self->{service_config}}) {
 457         my $d = $self->{service_config}->{$sid};
 458
 459         $w = Gtk3::Label->new($sid);
 460         $sgrid->attach($w, 0, $row, 1, 1);
 461
 462         $w = Gtk3::Switch->new();
 463         $sgrid->attach($w, 1, $row, 1, 1);
 464         $w->set_active(1) if $d->{state} eq 'enabled';
 465         $self->{service_gui}->{$sid}->{enable_btn} = $w;
 466         $w->signal_connect('notify::active' => sub {
 467             $self->set_service_state($sid);
 468         }),
 469
 470
 471         $w = Gtk3::Button->new('Migrate');
 472         $sgrid->attach($w, 2, $row, 1, 1);
 473         $w->signal_connect(clicked => sub {
 474             $self->show_migrate_dialog($sid);
 475         });
 476
 477         $w = Gtk3::Label->new($d->{node});
 478         $sgrid->attach($w, 3, $row, 1, 1);
 479         $self->{service_gui}->{$sid}->{node_label} = $w;
 480
 481         $w = Gtk3::Label->new('-');
 482         $w->set_alignment (0, 0.5);
 483         $sgrid->attach($w, 4, $row, 1, 1);
 484         $self->{service_gui}->{$sid}->{status_label} = $w;
 485
 486         $row++;
 487     }
 488
 489     return $sgrid;
 490 }
 491
 492 sub create_log_view {
 493     my ($self) = @_;
 494
 495     my $nb = Gtk3::Notebook->new();
 496
 497     my $l1 = Gtk3::Label->new('Cluster Log');
 498
 499     my $logview = Gtk3::TextView->new();
 500     $logview->set_editable(0);
 501     $logview->set_cursor_visible(0);
 502
 503     $self->{gui}->{text_view} = $logview;
 504
 505     my $swindow = Gtk3::ScrolledWindow->new();
 506     $swindow->set_size_request(1024, 768);
 507     $swindow->add($logview);
 508
 509     $nb->insert_page($swindow, $l1, 0);
 510
 511     my $l2 = Gtk3::Label->new('Manager Status');
 512
 513     my $statview = Gtk3::TextView->new();
 514     $statview->set_editable(0);
 515     $statview->set_cursor_visible(0);
 516
 517     $self->{gui}->{stat_view} = $statview;
 518
 519     $swindow = Gtk3::ScrolledWindow->new();
 520     $swindow->set_size_request(640, 400);
 521     $swindow->add($statview);
 522
 523     $nb->insert_page($swindow, $l2, 1);
 524     return $nb;
 525 }
 526
 527 sub create_main_window {
 528     my ($self) = @_;
 529
 530     my $window = Gtk3::Window->new();
 531     $window->set_title("Proxmox HA Simulator");
 532
 533     $window->signal_connect( destroy => sub { Gtk3::main_quit(); });
 534
 535     my $grid = Gtk3::Grid->new();
 536
 537     my $frame = $self->create_log_view();
 538     $grid->attach($frame, 0, 0, 1, 1);
 539     $frame->set('expand', 1);
 540
 541     my $vbox = Gtk3::VBox->new(0, 0);
 542     $grid->attach($vbox, 1, 0, 1, 1);
 543
 544     my $ngrid = $self->create_node_control();
 545     $vbox->pack_start($ngrid, 0, 0, 0);
 546
 547     my $sep = Gtk3::HSeparator->new;
 548     $sep->set('margin-top', 10);
 549     $vbox->pack_start ($sep, 0, 0, 0);
 550
 551     my $sgrid = $self->create_service_control();
 552     $vbox->pack_start($sgrid, 0, 0, 0);
 553
 554     $window->add($grid);
 555
 556     $window->show_all;
 557     $window->realize ();
 558 }
 559
 560 sub run {
 561     my ($self) = @_;
 562
 563     Glib::Timeout->add(1000, sub {
 564
 565         $self->{service_config} = $self->read_service_config();
 566
 567         # check all watchdogs
 568         my @nodes = sort keys %{$self->{nodes}};
 569         foreach my $node (@nodes) {
 570             if (!$self->watchdog_check($node)) {
 571                 $self->sim_hardware_cmd("power $node off", 'watchdog');
 572                 $self->log('info', "server '$node' stopped by poweroff (watchdog)");
 573             }
 574         }
 575
 576         my $mstatus = $self->read_manager_status();
 577         my $node_status = $mstatus->{node_status} || {};
 578
 579         foreach my $node (@nodes) {
 580             my $ns = $node_status->{$node} || '-';
 581             my $d = $self->{nodes}->{$node};
 582             next if !$d;
 583             my $sl = $d->{node_status_label};
 584             next if !$sl;
 585
 586             if ($mstatus->{master_node} && ($mstatus->{master_node} eq $node)) {
 587                 $sl->set_text(uc($ns));
 588             } else {
 589                 $sl->set_text($ns);
 590             }
 591         }
 592
 593         my $service_status = $mstatus->{service_status} || {};
 594         my @services = sort keys %{$self->{service_config}};
 595
 596         foreach my $sid (@services) {
 597             my $sc = $self->{service_config}->{$sid};
 598             my $ss = $service_status->{$sid};
 599             my $sgui = $self->{service_gui}->{$sid};
 600             next if !$sgui;
 601             my $nl = $sgui->{node_label};
 602             $nl->set_text($sc->{node});
 603
 604             my $sl = $sgui->{status_label};
 605             next if !$sl;
 606
 607             my $text = ($ss && $ss->{state}) ? $ss->{state} : '-';
 608             $sl->set_text($text);
 609         }
 610
 611         if (my $sv = $self->{gui}->{stat_view}) {
 612             my $text = Dumper($mstatus);
 613             my $textbuf = $sv->get_buffer();
 614             $textbuf->set_text($text, -1);
 615         }
 616
 617         return 1; # repeat
 618     });
 619
 620     Gtk3->main;
 621
 622     $self->cleanup();
 623 }
 624
 625 1;