ceph/src/pybind/mgr/dashboard/frontend/src/app/shared/services/prometheus-alert.service.ts

   1 import { Injectable } from '@angular/core';
   2
   3 import _ from 'lodash';
   4
   5 import { PrometheusService } from '../api/prometheus.service';
   6 import {
   7   AlertmanagerAlert,
   8   PrometheusCustomAlert,
   9   PrometheusRule
  10 } from '../models/prometheus-alerts';
  11 import { PrometheusAlertFormatter } from './prometheus-alert-formatter';
  12
  13 @Injectable({
  14   providedIn: 'root'
  15 })
  16 export class PrometheusAlertService {
  17   private canAlertsBeNotified = false;
  18   alerts: AlertmanagerAlert[] = [];
  19   rules: PrometheusRule[] = [];
  20   activeAlerts: number;
  21   activeCriticalAlerts: number;
  22   activeWarningAlerts: number;
  23
  24   constructor(
  25     private alertFormatter: PrometheusAlertFormatter,
  26     private prometheusService: PrometheusService
  27   ) {}
  28
  29   getAlerts() {
  30     this.prometheusService.ifAlertmanagerConfigured(() => {
  31       this.prometheusService.getAlerts().subscribe(
  32         (alerts) => this.handleAlerts(alerts),
  33         (resp) => {
  34           if ([404, 504].includes(resp.status)) {
  35             this.prometheusService.disableAlertmanagerConfig();
  36           }
  37         }
  38       );
  39     });
  40   }
  41
  42   getRules() {
  43     this.prometheusService.ifPrometheusConfigured(() => {
  44       this.prometheusService.getRules('alerting').subscribe((groups) => {
  45         this.rules = groups['groups'].reduce((acc, group) => {
  46           return acc.concat(
  47             group.rules.map((rule) => {
  48               rule.group = group.name;
  49               return rule;
  50             })
  51           );
  52         }, []);
  53       });
  54     });
  55   }
  56
  57   refresh() {
  58     this.getAlerts();
  59     this.getRules();
  60   }
  61
  62   private handleAlerts(alerts: AlertmanagerAlert[]) {
  63     if (this.canAlertsBeNotified) {
  64       this.notifyOnAlertChanges(alerts, this.alerts);
  65     }
  66     this.activeAlerts = _.reduce<AlertmanagerAlert, number>(
  67       this.alerts,
  68       (result, alert) => (alert.status.state === 'active' ? ++result : result),
  69       0
  70     );
  71     this.activeCriticalAlerts = _.reduce<AlertmanagerAlert, number>(
  72       this.alerts,
  73       (result, alert) =>
  74         alert.status.state === 'active' && alert.labels.severity === 'critical' ? ++result : result,
  75       0
  76     );
  77     this.activeWarningAlerts = _.reduce<AlertmanagerAlert, number>(
  78       this.alerts,
  79       (result, alert) =>
  80         alert.status.state === 'active' && alert.labels.severity === 'warning' ? ++result : result,
  81       0
  82     );
  83     this.alerts = alerts;
  84     this.canAlertsBeNotified = true;
  85   }
  86
  87   private notifyOnAlertChanges(alerts: AlertmanagerAlert[], oldAlerts: AlertmanagerAlert[]) {
  88     const changedAlerts = this.getChangedAlerts(
  89       this.alertFormatter.convertToCustomAlerts(alerts),
  90       this.alertFormatter.convertToCustomAlerts(oldAlerts)
  91     );
  92     const suppressedFiltered = _.filter(changedAlerts, (alert) => {
  93       return alert.status !== 'suppressed';
  94     });
  95     const notifications = suppressedFiltered.map((alert) =>
  96       this.alertFormatter.convertAlertToNotification(alert)
  97     );
  98     this.alertFormatter.sendNotifications(notifications);
  99   }
 100
 101   private getChangedAlerts(alerts: PrometheusCustomAlert[], oldAlerts: PrometheusCustomAlert[]) {
 102     const updatedAndNew = _.differenceWith(alerts, oldAlerts, _.isEqual);
 103     return updatedAndNew.concat(this.getVanishedAlerts(alerts, oldAlerts));
 104   }
 105
 106   private getVanishedAlerts(alerts: PrometheusCustomAlert[], oldAlerts: PrometheusCustomAlert[]) {
 107     return _.differenceWith(oldAlerts, alerts, (a, b) => a.fingerprint === b.fingerprint).map(
 108       (alert) => {
 109         alert.status = 'resolved';
 110         return alert;
 111       }
 112     );
 113   }
 114 }