src/server/notifications.rs

   1 use anyhow::Error;
   2 use const_format::concatcp;
   3 use serde_json::json;
   4 use std::collections::HashMap;
   5 use std::path::Path;
   6 use std::time::{Duration, Instant};
   7
   8 use handlebars::{Handlebars, TemplateError};
   9 use nix::unistd::Uid;
  10
  11 use proxmox_lang::try_block;
  12 use proxmox_notify::context::pbs::PBS_CONTEXT;
  13 use proxmox_schema::ApiType;
  14 use proxmox_sys::email::sendmail;
  15 use proxmox_sys::fs::{create_path, CreateOptions};
  16
  17 use pbs_api_types::{
  18     APTUpdateInfo, DataStoreConfig, DatastoreNotify, GarbageCollectionStatus, NotificationMode,
  19     Notify, SyncJobConfig, TapeBackupJobSetup, User, Userid, VerificationJobConfig,
  20 };
  21 use proxmox_notify::endpoints::sendmail::{SendmailConfig, SendmailEndpoint};
  22 use proxmox_notify::{Endpoint, Notification, Severity};
  23
  24 const SPOOL_DIR: &str = concatcp!(pbs_buildcfg::PROXMOX_BACKUP_STATE_DIR, "/notifications");
  25
  26 const TAPE_BACKUP_OK_TEMPLATE: &str = r###"
  27
  28 {{#if id ~}}
  29 Job ID:     {{id}}
  30 {{/if~}}
  31 Datastore:  {{job.store}}
  32 Tape Pool:  {{job.pool}}
  33 Tape Drive: {{job.drive}}
  34
  35 {{#if snapshot-list ~}}
  36 Snapshots included:
  37
  38 {{#each snapshot-list~}}
  39 {{this}}
  40 {{/each~}}
  41 {{/if}}
  42 Duration: {{duration}}
  43 {{#if used-tapes }}
  44 Used Tapes:
  45 {{#each used-tapes~}}
  46 {{this}}
  47 {{/each~}}
  48 {{/if}}
  49 Tape Backup successful.
  50
  51
  52 Please visit the web interface for further details:
  53
  54 <https://{{fqdn}}:{{port}}/#DataStore-{{job.store}}>
  55
  56 "###;
  57
  58 const TAPE_BACKUP_ERR_TEMPLATE: &str = r###"
  59
  60 {{#if id ~}}
  61 Job ID:     {{id}}
  62 {{/if~}}
  63 Datastore:  {{job.store}}
  64 Tape Pool:  {{job.pool}}
  65 Tape Drive: {{job.drive}}
  66
  67 {{#if snapshot-list ~}}
  68 Snapshots included:
  69
  70 {{#each snapshot-list~}}
  71 {{this}}
  72 {{/each~}}
  73 {{/if}}
  74 {{#if used-tapes }}
  75 Used Tapes:
  76 {{#each used-tapes~}}
  77 {{this}}
  78 {{/each~}}
  79 {{/if}}
  80 Tape Backup failed: {{error}}
  81
  82
  83 Please visit the web interface for further details:
  84
  85 <https://{{fqdn}}:{{port}}/#pbsServerAdministration:tasks>
  86
  87 "###;
  88
  89 lazy_static::lazy_static! {
  90
  91     static ref HANDLEBARS: Handlebars<'static> = {
  92         let mut hb = Handlebars::new();
  93         let result: Result<(), TemplateError> = try_block!({
  94
  95             hb.set_strict_mode(true);
  96             hb.register_escape_fn(handlebars::no_escape);
  97
  98             hb.register_template_string("tape_backup_ok_template", TAPE_BACKUP_OK_TEMPLATE)?;
  99             hb.register_template_string("tape_backup_err_template", TAPE_BACKUP_ERR_TEMPLATE)?;
 100
 101             Ok(())
 102         });
 103
 104         if let Err(err) = result {
 105             eprintln!("error during template registration: {err}");
 106         }
 107
 108         hb
 109     };
 110 }
 111
 112 /// Initialize the notification system by setting context in proxmox_notify
 113 pub fn init() -> Result<(), Error> {
 114     proxmox_notify::context::set_context(&PBS_CONTEXT);
 115     Ok(())
 116 }
 117
 118 /// Create the directory which will be used to temporarily store notifications
 119 /// which were sent from an unprivileged process.
 120 pub fn create_spool_dir() -> Result<(), Error> {
 121     let backup_user = pbs_config::backup_user()?;
 122     let opts = CreateOptions::new()
 123         .owner(backup_user.uid)
 124         .group(backup_user.gid);
 125
 126     create_path(SPOOL_DIR, None, Some(opts))?;
 127     Ok(())
 128 }
 129
 130 async fn send_queued_notifications() -> Result<(), Error> {
 131     let mut read_dir = tokio::fs::read_dir(SPOOL_DIR).await?;
 132
 133     let mut notifications = Vec::new();
 134
 135     while let Some(entry) = read_dir.next_entry().await? {
 136         let path = entry.path();
 137
 138         if let Some(ext) = path.extension() {
 139             if ext == "json" {
 140                 let p = path.clone();
 141
 142                 let bytes = tokio::fs::read(p).await?;
 143                 let notification: Notification = serde_json::from_slice(&bytes)?;
 144                 notifications.push(notification);
 145
 146                 // Currently, there is no retry-mechanism in case of failure...
 147                 // For retries, we'd have to keep track of which targets succeeded/failed
 148                 // to send, so we do not retry notifying a target which succeeded before.
 149                 tokio::fs::remove_file(path).await?;
 150             }
 151         }
 152     }
 153
 154     // Make sure that we send the oldest notification first
 155     notifications.sort_unstable_by_key(|n| n.timestamp());
 156
 157     let res = tokio::task::spawn_blocking(move || {
 158         let config = pbs_config::notifications::config()?;
 159         for notification in notifications {
 160             if let Err(err) = proxmox_notify::api::common::send(&config, &notification) {
 161                 log::error!("failed to send notification: {err}");
 162             }
 163         }
 164
 165         Ok::<(), Error>(())
 166     })
 167     .await?;
 168
 169     if let Err(e) = res {
 170         log::error!("could not read notification config: {e}");
 171     }
 172
 173     Ok::<(), Error>(())
 174 }
 175
 176 /// Worker task to periodically send any queued notifications.
 177 pub async fn notification_worker() {
 178     loop {
 179         let delay_target = Instant::now() + Duration::from_secs(5);
 180
 181         if let Err(err) = send_queued_notifications().await {
 182             log::error!("notification worker task error: {err}");
 183         }
 184
 185         tokio::time::sleep_until(tokio::time::Instant::from_std(delay_target)).await;
 186     }
 187 }
 188
 189 fn send_notification(notification: Notification) -> Result<(), Error> {
 190     if nix::unistd::ROOT == Uid::current() {
 191         let config = pbs_config::notifications::config()?;
 192         proxmox_notify::api::common::send(&config, &notification)?;
 193     } else {
 194         let ser = serde_json::to_vec(&notification)?;
 195         let path = Path::new(SPOOL_DIR).join(format!("{id}.json", id = notification.id()));
 196
 197         let backup_user = pbs_config::backup_user()?;
 198         let opts = CreateOptions::new()
 199             .owner(backup_user.uid)
 200             .group(backup_user.gid);
 201         proxmox_sys::fs::replace_file(path, &ser, opts, true)?;
 202         log::info!("queued notification (id={id})", id = notification.id())
 203     }
 204
 205     Ok(())
 206 }
 207
 208 fn send_sendmail_legacy_notification(notification: Notification, email: &str) -> Result<(), Error> {
 209     let endpoint = SendmailEndpoint {
 210         config: SendmailConfig {
 211             mailto: vec![email.into()],
 212             ..Default::default()
 213         },
 214     };
 215
 216     endpoint.send(&notification)?;
 217
 218     Ok(())
 219 }
 220
 221 /// Summary of a successful Tape Job
 222 #[derive(Default)]
 223 pub struct TapeBackupJobSummary {
 224     /// The list of snaphots backed up
 225     pub snapshot_list: Vec<String>,
 226     /// The total time of the backup job
 227     pub duration: std::time::Duration,
 228     /// The labels of the used tapes of the backup job
 229     pub used_tapes: Option<Vec<String>>,
 230 }
 231
 232 fn send_job_status_mail(email: &str, subject: &str, text: &str) -> Result<(), Error> {
 233     let (config, _) = crate::config::node::config()?;
 234     let from = config.email_from;
 235
 236     // NOTE: some (web)mailers have big problems displaying text mails, so include html as well
 237     let escaped_text = handlebars::html_escape(text);
 238     let html = format!("<html><body><pre>\n{escaped_text}\n<pre>");
 239
 240     let nodename = proxmox_sys::nodename();
 241
 242     let author = format!("Proxmox Backup Server - {nodename}");
 243
 244     sendmail(
 245         &[email],
 246         subject,
 247         Some(text),
 248         Some(&html),
 249         from.as_deref(),
 250         Some(&author),
 251     )?;
 252
 253     Ok(())
 254 }
 255
 256 pub fn send_gc_status(
 257     datastore: &str,
 258     status: &GarbageCollectionStatus,
 259     result: &Result<(), Error>,
 260 ) -> Result<(), Error> {
 261     let (fqdn, port) = get_server_url();
 262     let mut data = json!({
 263         "datastore": datastore,
 264         "fqdn": fqdn,
 265         "port": port,
 266     });
 267
 268     let (severity, template) = match result {
 269         Ok(()) => {
 270             let deduplication_factor = if status.disk_bytes > 0 {
 271                 (status.index_data_bytes as f64) / (status.disk_bytes as f64)
 272             } else {
 273                 1.0
 274             };
 275
 276             data["status"] = json!(status);
 277             data["deduplication-factor"] = format!("{:.2}", deduplication_factor).into();
 278
 279             (Severity::Info, "gc-ok")
 280         }
 281         Err(err) => {
 282             data["error"] = err.to_string().into();
 283             (Severity::Error, "gc-err")
 284         }
 285     };
 286     let metadata = HashMap::from([
 287         ("datastore".into(), datastore.into()),
 288         ("hostname".into(), proxmox_sys::nodename().into()),
 289         ("type".into(), "gc".into()),
 290     ]);
 291
 292     let notification = Notification::from_template(severity, template, data, metadata);
 293
 294     let (email, notify, mode) = lookup_datastore_notify_settings(datastore);
 295     match mode {
 296         NotificationMode::LegacySendmail => {
 297             let notify = notify.gc.unwrap_or(Notify::Always);
 298
 299             if notify == Notify::Never || (result.is_ok() && notify == Notify::Error) {
 300                 return Ok(());
 301             }
 302
 303             if let Some(email) = email {
 304                 send_sendmail_legacy_notification(notification, &email)?;
 305             }
 306         }
 307         NotificationMode::NotificationSystem => {
 308             send_notification(notification)?;
 309         }
 310     }
 311
 312     Ok(())
 313 }
 314
 315 pub fn send_verify_status(
 316     job: VerificationJobConfig,
 317     result: &Result<Vec<String>, Error>,
 318 ) -> Result<(), Error> {
 319     let (fqdn, port) = get_server_url();
 320     let mut data = json!({
 321         "job": job,
 322         "fqdn": fqdn,
 323         "port": port,
 324     });
 325
 326     let (template, severity) = match result {
 327         Ok(errors) if errors.is_empty() => ("verify-ok", Severity::Info),
 328         Ok(errors) => {
 329             data["errors"] = json!(errors);
 330             ("verify-err", Severity::Error)
 331         }
 332         Err(_) => {
 333             // aborted job - do not send any notification
 334             return Ok(());
 335         }
 336     };
 337
 338     let metadata = HashMap::from([
 339         ("job-id".into(), job.id.clone()),
 340         ("datastore".into(), job.store.clone()),
 341         ("hostname".into(), proxmox_sys::nodename().into()),
 342         ("type".into(), "verify".into()),
 343     ]);
 344
 345     let notification = Notification::from_template(severity, template, data, metadata);
 346
 347     let (email, notify, mode) = lookup_datastore_notify_settings(&job.store);
 348     match mode {
 349         NotificationMode::LegacySendmail => {
 350             let notify = notify.verify.unwrap_or(Notify::Always);
 351
 352             if notify == Notify::Never || (result.is_ok() && notify == Notify::Error) {
 353                 return Ok(());
 354             }
 355
 356             if let Some(email) = email {
 357                 send_sendmail_legacy_notification(notification, &email)?;
 358             }
 359         }
 360         NotificationMode::NotificationSystem => {
 361             send_notification(notification)?;
 362         }
 363     }
 364
 365     Ok(())
 366 }
 367
 368 pub fn send_prune_status(
 369     store: &str,
 370     jobname: &str,
 371     result: &Result<(), Error>,
 372 ) -> Result<(), Error> {
 373     let (fqdn, port) = get_server_url();
 374     let mut data = json!({
 375         "jobname": jobname,
 376         "store": store,
 377         "fqdn": fqdn,
 378         "port": port,
 379     });
 380
 381     let (template, severity) = match result {
 382         Ok(()) => ("prune-ok", Severity::Info),
 383         Err(err) => {
 384             data["error"] = err.to_string().into();
 385             ("prune-err", Severity::Error)
 386         }
 387     };
 388
 389     let metadata = HashMap::from([
 390         ("job-id".into(), jobname.to_string()),
 391         ("datastore".into(), store.into()),
 392         ("hostname".into(), proxmox_sys::nodename().into()),
 393         ("type".into(), "prune".into()),
 394     ]);
 395
 396     let notification = Notification::from_template(severity, template, data, metadata);
 397
 398     let (email, notify, mode) = lookup_datastore_notify_settings(store);
 399     match mode {
 400         NotificationMode::LegacySendmail => {
 401             let notify = notify.prune.unwrap_or(Notify::Error);
 402
 403             if notify == Notify::Never || (result.is_ok() && notify == Notify::Error) {
 404                 return Ok(());
 405             }
 406
 407             if let Some(email) = email {
 408                 send_sendmail_legacy_notification(notification, &email)?;
 409             }
 410         }
 411         NotificationMode::NotificationSystem => {
 412             send_notification(notification)?;
 413         }
 414     }
 415
 416     Ok(())
 417 }
 418
 419 pub fn send_sync_status(job: &SyncJobConfig, result: &Result<(), Error>) -> Result<(), Error> {
 420     let (fqdn, port) = get_server_url();
 421     let mut data = json!({
 422         "job": job,
 423         "fqdn": fqdn,
 424         "port": port,
 425     });
 426
 427     let (template, severity) = match result {
 428         Ok(()) => ("sync-ok", Severity::Info),
 429         Err(err) => {
 430             data["error"] = err.to_string().into();
 431             ("sync-err", Severity::Error)
 432         }
 433     };
 434
 435     let metadata = HashMap::from([
 436         ("job-id".into(), job.id.clone()),
 437         ("datastore".into(), job.store.clone()),
 438         ("hostname".into(), proxmox_sys::nodename().into()),
 439         ("type".into(), "sync".into()),
 440     ]);
 441
 442     let notification = Notification::from_template(severity, template, data, metadata);
 443
 444     let (email, notify, mode) = lookup_datastore_notify_settings(&job.store);
 445     match mode {
 446         NotificationMode::LegacySendmail => {
 447             let notify = notify.prune.unwrap_or(Notify::Error);
 448
 449             if notify == Notify::Never || (result.is_ok() && notify == Notify::Error) {
 450                 return Ok(());
 451             }
 452
 453             if let Some(email) = email {
 454                 send_sendmail_legacy_notification(notification, &email)?;
 455             }
 456         }
 457         NotificationMode::NotificationSystem => {
 458             send_notification(notification)?;
 459         }
 460     }
 461
 462     Ok(())
 463 }
 464
 465 pub fn send_tape_backup_status(
 466     email: &str,
 467     id: Option<&str>,
 468     job: &TapeBackupJobSetup,
 469     result: &Result<(), Error>,
 470     summary: TapeBackupJobSummary,
 471 ) -> Result<(), Error> {
 472     let (fqdn, port) = get_server_url();
 473     let duration: proxmox_time::TimeSpan = summary.duration.into();
 474     let mut data = json!({
 475         "job": job,
 476         "fqdn": fqdn,
 477         "port": port,
 478         "id": id,
 479         "snapshot-list": summary.snapshot_list,
 480         "used-tapes": summary.used_tapes,
 481         "duration": duration.to_string(),
 482     });
 483
 484     let text = match result {
 485         Ok(()) => HANDLEBARS.render("tape_backup_ok_template", &data)?,
 486         Err(err) => {
 487             data["error"] = err.to_string().into();
 488             HANDLEBARS.render("tape_backup_err_template", &data)?
 489         }
 490     };
 491
 492     let subject = match (result, id) {
 493         (Ok(()), Some(id)) => format!("Tape Backup '{id}' datastore '{}' successful", job.store,),
 494         (Ok(()), None) => format!("Tape Backup datastore '{}' successful", job.store,),
 495         (Err(_), Some(id)) => format!("Tape Backup '{id}' datastore '{}' failed", job.store,),
 496         (Err(_), None) => format!("Tape Backup datastore '{}' failed", job.store,),
 497     };
 498
 499     send_job_status_mail(email, &subject, &text)?;
 500
 501     Ok(())
 502 }
 503
 504 /// Send email to a person to request a manual media change
 505 pub fn send_load_media_email(
 506     changer: bool,
 507     device: &str,
 508     label_text: &str,
 509     to: &str,
 510     reason: Option<String>,
 511 ) -> Result<(), Error> {
 512     use std::fmt::Write as _;
 513
 514     let device_type = if changer { "changer" } else { "drive" };
 515
 516     let subject = format!("Load Media '{label_text}' request for {device_type} '{device}'");
 517
 518     let mut text = String::new();
 519
 520     if let Some(reason) = reason {
 521         let _ = write!(
 522             text,
 523             "The {device_type} has the wrong or no tape(s) inserted. Error:\n{reason}\n\n"
 524         );
 525     }
 526
 527     if changer {
 528         text.push_str("Please insert the requested media into the changer.\n\n");
 529         let _ = writeln!(text, "Changer: {device}");
 530     } else {
 531         text.push_str("Please insert the requested media into the backup drive.\n\n");
 532         let _ = writeln!(text, "Drive: {device}");
 533     }
 534     let _ = writeln!(text, "Media: {label_text}");
 535
 536     send_job_status_mail(to, &subject, &text)
 537 }
 538
 539 fn get_server_url() -> (String, usize) {
 540     // user will surely request that they can change this
 541
 542     let nodename = proxmox_sys::nodename();
 543     let mut fqdn = nodename.to_owned();
 544
 545     if let Ok(resolv_conf) = crate::api2::node::dns::read_etc_resolv_conf() {
 546         if let Some(search) = resolv_conf["search"].as_str() {
 547             fqdn.push('.');
 548             fqdn.push_str(search);
 549         }
 550     }
 551
 552     let port = 8007;
 553
 554     (fqdn, port)
 555 }
 556
 557 pub fn send_updates_available(updates: &[&APTUpdateInfo]) -> Result<(), Error> {
 558     let (fqdn, port) = get_server_url();
 559     let hostname = proxmox_sys::nodename().to_string();
 560
 561     let data = json!({
 562         "fqdn": fqdn,
 563         "hostname": &hostname,
 564         "port": port,
 565         "updates": updates,
 566     });
 567
 568     let metadata = HashMap::from([
 569         ("hostname".into(), hostname),
 570         ("type".into(), "package-updates".into()),
 571     ]);
 572
 573     let notification =
 574         Notification::from_template(Severity::Info, "package-updates", data, metadata);
 575
 576     send_notification(notification)?;
 577     Ok(())
 578 }
 579
 580 /// send email on certificate renewal failure.
 581 pub fn send_certificate_renewal_mail(result: &Result<(), Error>) -> Result<(), Error> {
 582     let error: String = match result {
 583         Err(e) => e.to_string(),
 584         _ => return Ok(()),
 585     };
 586
 587     let (fqdn, port) = get_server_url();
 588
 589     let data = json!({
 590         "fqdn": fqdn,
 591         "port": port,
 592         "error": error,
 593     });
 594
 595     let metadata = HashMap::from([
 596         ("hostname".into(), proxmox_sys::nodename().into()),
 597         ("type".into(), "acme".into()),
 598     ]);
 599
 600     let notification = Notification::from_template(Severity::Info, "acme-err", data, metadata);
 601
 602     send_notification(notification)?;
 603     Ok(())
 604 }
 605
 606 /// Lookup users email address
 607 pub fn lookup_user_email(userid: &Userid) -> Option<String> {
 608     if let Ok(user_config) = pbs_config::user::cached_config() {
 609         if let Ok(user) = user_config.lookup::<User>("user", userid.as_str()) {
 610             return user.email;
 611         }
 612     }
 613
 614     None
 615 }
 616
 617 /// Lookup Datastore notify settings
 618 pub fn lookup_datastore_notify_settings(
 619     store: &str,
 620 ) -> (Option<String>, DatastoreNotify, NotificationMode) {
 621     let mut email = None;
 622
 623     let notify = DatastoreNotify {
 624         gc: None,
 625         verify: None,
 626         sync: None,
 627         prune: None,
 628     };
 629
 630     let (config, _digest) = match pbs_config::datastore::config() {
 631         Ok(result) => result,
 632         Err(_) => return (email, notify, NotificationMode::default()),
 633     };
 634
 635     let config: DataStoreConfig = match config.lookup("datastore", store) {
 636         Ok(result) => result,
 637         Err(_) => return (email, notify, NotificationMode::default()),
 638     };
 639
 640     email = match config.notify_user {
 641         Some(ref userid) => lookup_user_email(userid),
 642         None => lookup_user_email(Userid::root_userid()),
 643     };
 644
 645     let notification_mode = config.notification_mode.unwrap_or_default();
 646     let notify_str = config.notify.unwrap_or_default();
 647
 648     if let Ok(value) = DatastoreNotify::API_SCHEMA.parse_property_string(&notify_str) {
 649         if let Ok(notify) = serde_json::from_value(value) {
 650             return (email, notify, notification_mode);
 651         }
 652     }
 653
 654     (email, notify, notification_mode)
 655 }
 656
 657 #[test]
 658 fn test_template_register() {
 659     assert!(HANDLEBARS.has_template("tape_backup_ok_template"));
 660     assert!(HANDLEBARS.has_template("tape_backup_err_template"));
 661 }