proxmox-rrd/src/rrd.rs

   1 //! # Round Robin Database file format
   2
   3 use std::io::Read;
   4 use std::path::Path;
   5
   6 use anyhow::{bail, Error};
   7 use bitflags::bitflags;
   8
   9 use proxmox::tools::{fs::replace_file, fs::CreateOptions};
  10
  11 use proxmox_rrd_api_types::{RRDMode, RRDTimeFrameResolution};
  12
  13 /// The number of data entries per RRA
  14 pub const RRD_DATA_ENTRIES: usize = 70;
  15
  16 /// Proxmox RRD file magic number
  17 // openssl::sha::sha256(b"Proxmox Round Robin Database file v1.0")[0..8];
  18 pub const PROXMOX_RRD_MAGIC_1_0: [u8; 8] =  [206, 46, 26, 212, 172, 158, 5, 186];
  19
  20 use crate::DST;
  21
  22 bitflags!{
  23     /// Flags to specify the data soure type and consolidation function
  24     pub struct RRAFlags: u64 {
  25         // Data Source Types
  26         const DST_GAUGE  = 1;
  27         const DST_DERIVE = 2;
  28         const DST_COUNTER = 4;
  29         const DST_MASK   = 255; // first 8 bits
  30
  31         // Consolidation Functions
  32         const CF_AVERAGE = 1 << 8;
  33         const CF_MAX     = 2 << 8;
  34         const CF_MASK    = 255 << 8;
  35     }
  36 }
  37
  38 /// Round Robin Archive with [RRD_DATA_ENTRIES] data slots.
  39 ///
  40 /// This data structure is used inside [RRD] and directly written to the
  41 /// RRD files.
  42 #[repr(C)]
  43 pub struct RRA {
  44     /// Defined the data soure type and consolidation function
  45     pub flags: RRAFlags,
  46     /// Resulution (seconds) from [RRDTimeFrameResolution]
  47     pub resolution: u64,
  48     /// Last update time (epoch)
  49     pub last_update: f64,
  50     /// Count values computed inside this update interval
  51     pub last_count: u64,
  52     /// Stores the last value, used to compute differential value for derive/counters
  53     pub counter_value: f64,
  54     /// Data slots
  55     pub data: [f64; RRD_DATA_ENTRIES],
  56 }
  57
  58 impl RRA {
  59     fn new(flags: RRAFlags, resolution: u64) -> Self {
  60         Self {
  61             flags, resolution,
  62             last_update: 0.0,
  63             last_count: 0,
  64             counter_value: f64::NAN,
  65             data: [f64::NAN; RRD_DATA_ENTRIES],
  66         }
  67     }
  68
  69     fn delete_old(&mut self, time: f64) {
  70         let epoch = time as u64;
  71         let last_update = self.last_update as u64;
  72         let reso = self.resolution;
  73
  74         let min_time = epoch - (RRD_DATA_ENTRIES as u64)*reso;
  75         let min_time = (min_time/reso + 1)*reso;
  76         let mut t = last_update.saturating_sub((RRD_DATA_ENTRIES as u64)*reso);
  77         let mut index = ((t/reso) % (RRD_DATA_ENTRIES as u64)) as usize;
  78         for _ in 0..RRD_DATA_ENTRIES {
  79             t += reso; index = (index + 1) % RRD_DATA_ENTRIES;
  80             if t < min_time {
  81                 self.data[index] = f64::NAN;
  82             } else {
  83                 break;
  84             }
  85         }
  86     }
  87
  88     fn compute_new_value(&mut self, time: f64, value: f64) {
  89         let epoch = time as u64;
  90         let last_update = self.last_update as u64;
  91         let reso = self.resolution;
  92
  93         let index = ((epoch/reso) % (RRD_DATA_ENTRIES as u64)) as usize;
  94         let last_index = ((last_update/reso) % (RRD_DATA_ENTRIES as u64)) as usize;
  95
  96         if (epoch - (last_update as u64)) > reso || index != last_index {
  97             self.last_count = 0;
  98         }
  99
 100         let last_value = self.data[index];
 101         if last_value.is_nan() {
 102             self.last_count = 0;
 103         }
 104
 105         let new_count = if self.last_count < u64::MAX {
 106             self.last_count + 1
 107         } else {
 108             u64::MAX // should never happen
 109         };
 110
 111         if self.last_count == 0 {
 112             self.data[index] = value;
 113             self.last_count = 1;
 114         } else {
 115             let new_value = if self.flags.contains(RRAFlags::CF_MAX) {
 116                 if last_value > value { last_value } else { value }
 117             } else if self.flags.contains(RRAFlags::CF_AVERAGE) {
 118                 (last_value*(self.last_count as f64))/(new_count as f64)
 119                     + value/(new_count as f64)
 120             } else {
 121                 log::error!("rrdb update failed - unknown CF");
 122                 return;
 123             };
 124             self.data[index] = new_value;
 125             self.last_count = new_count;
 126         }
 127         self.last_update = time;
 128     }
 129
 130     // Note: This may update the state even in case of errors (see counter overflow)
 131     fn update(&mut self, time: f64, mut value: f64) -> Result<(), Error> {
 132
 133         if time <= self.last_update {
 134             bail!("time in past ({} < {})", time, self.last_update);
 135         }
 136
 137         if value.is_nan() {
 138             bail!("new value is NAN");
 139         }
 140
 141         // derive counter value
 142         if self.flags.intersects(RRAFlags::DST_DERIVE | RRAFlags::DST_COUNTER) {
 143             let time_diff = time - self.last_update;
 144             let is_counter = self.flags.contains(RRAFlags::DST_COUNTER);
 145
 146             let diff = if self.counter_value.is_nan() {
 147                 0.0
 148             } else if is_counter && value < 0.0 {
 149                 bail!("got negative value for counter");
 150             } else if is_counter && value < self.counter_value {
 151                 // Note: We do not try automatic overflow corrections, but
 152                 // we update counter_value anyways, so that we can compute the diff
 153                 // next time.
 154                 self.counter_value = value;
 155                 bail!("conter overflow/reset detected");
 156             } else {
 157                 value - self.counter_value
 158             };
 159             self.counter_value = value;
 160             value = diff/time_diff;
 161         }
 162
 163         self.delete_old(time);
 164         self.compute_new_value(time, value);
 165
 166         Ok(())
 167     }
 168 }
 169
 170 /// Round Robin Database file format with fixed number of [RRA]s
 171 #[repr(C)]
 172 // Note: Avoid alignment problems by using 8byte types only
 173 pub struct RRD {
 174     /// The magic number to identify the file type
 175     pub magic: [u8; 8],
 176     /// Hourly data (average values)
 177     pub hour_avg: RRA,
 178     /// Hourly data (maximum values)
 179     pub hour_max: RRA,
 180     /// Dayly data (average values)
 181     pub day_avg: RRA,
 182     /// Dayly data (maximum values)
 183     pub day_max: RRA,
 184     /// Weekly data (average values)
 185     pub week_avg: RRA,
 186     /// Weekly data (maximum values)
 187     pub week_max: RRA,
 188     /// Monthly data (average values)
 189     pub month_avg: RRA,
 190     /// Monthly data (maximum values)
 191     pub month_max: RRA,
 192     /// Yearly data (average values)
 193     pub year_avg: RRA,
 194     /// Yearly data (maximum values)
 195     pub year_max: RRA,
 196 }
 197
 198 impl RRD {
 199
 200     /// Create a new empty instance
 201     pub fn new(dst: DST) -> Self {
 202         let flags = match dst {
 203             DST::Gauge => RRAFlags::DST_GAUGE,
 204             DST::Derive => RRAFlags::DST_DERIVE,
 205         };
 206
 207         Self {
 208             magic: PROXMOX_RRD_MAGIC_1_0,
 209             hour_avg: RRA::new(
 210                 flags | RRAFlags::CF_AVERAGE,
 211                 RRDTimeFrameResolution::Hour as u64,
 212             ),
 213             hour_max: RRA::new(
 214                 flags |  RRAFlags::CF_MAX,
 215                 RRDTimeFrameResolution::Hour as u64,
 216             ),
 217             day_avg: RRA::new(
 218                 flags |  RRAFlags::CF_AVERAGE,
 219                 RRDTimeFrameResolution::Day as u64,
 220             ),
 221             day_max: RRA::new(
 222                 flags |  RRAFlags::CF_MAX,
 223                 RRDTimeFrameResolution::Day as u64,
 224             ),
 225             week_avg: RRA::new(
 226                 flags |  RRAFlags::CF_AVERAGE,
 227                 RRDTimeFrameResolution::Week as u64,
 228             ),
 229             week_max: RRA::new(
 230                 flags |  RRAFlags::CF_MAX,
 231                 RRDTimeFrameResolution::Week as u64,
 232             ),
 233             month_avg: RRA::new(
 234                 flags |  RRAFlags::CF_AVERAGE,
 235                 RRDTimeFrameResolution::Month as u64,
 236             ),
 237             month_max: RRA::new(
 238                 flags |  RRAFlags::CF_MAX,
 239                 RRDTimeFrameResolution::Month as u64,
 240             ),
 241             year_avg: RRA::new(
 242                 flags |  RRAFlags::CF_AVERAGE,
 243                 RRDTimeFrameResolution::Year as u64,
 244             ),
 245             year_max: RRA::new(
 246                 flags |  RRAFlags::CF_MAX,
 247                 RRDTimeFrameResolution::Year as u64,
 248             ),
 249         }
 250     }
 251
 252     /// Extract data from the archive
 253     pub fn extract_data(
 254         &self,
 255         time: f64,
 256         timeframe: RRDTimeFrameResolution,
 257         mode: RRDMode,
 258     ) -> (u64, u64, Vec<Option<f64>>) {
 259
 260         let epoch = time as u64;
 261         let reso = timeframe as u64;
 262
 263         let end = reso*(epoch/reso + 1);
 264         let start = end - reso*(RRD_DATA_ENTRIES as u64);
 265
 266         let mut list = Vec::new();
 267
 268         let raa = match (mode, timeframe) {
 269             (RRDMode::Average, RRDTimeFrameResolution::Hour) => &self.hour_avg,
 270             (RRDMode::Max, RRDTimeFrameResolution::Hour) => &self.hour_max,
 271             (RRDMode::Average, RRDTimeFrameResolution::Day) => &self.day_avg,
 272             (RRDMode::Max, RRDTimeFrameResolution::Day) => &self.day_max,
 273             (RRDMode::Average, RRDTimeFrameResolution::Week) => &self.week_avg,
 274             (RRDMode::Max, RRDTimeFrameResolution::Week) => &self.week_max,
 275             (RRDMode::Average, RRDTimeFrameResolution::Month) => &self.month_avg,
 276             (RRDMode::Max, RRDTimeFrameResolution::Month) => &self.month_max,
 277             (RRDMode::Average, RRDTimeFrameResolution::Year) => &self.year_avg,
 278             (RRDMode::Max, RRDTimeFrameResolution::Year) => &self.year_max,
 279         };
 280
 281         let rrd_end = reso*((raa.last_update as u64)/reso);
 282         let rrd_start = rrd_end - reso*(RRD_DATA_ENTRIES as u64);
 283
 284         let mut t = start;
 285         let mut index = ((t/reso) % (RRD_DATA_ENTRIES as u64)) as usize;
 286         for _ in 0..RRD_DATA_ENTRIES {
 287             if t < rrd_start || t > rrd_end {
 288                 list.push(None);
 289             } else {
 290                 let value = raa.data[index];
 291                 if value.is_nan() {
 292                     list.push(None);
 293                 } else {
 294                     list.push(Some(value));
 295                 }
 296             }
 297             t += reso; index = (index + 1) % RRD_DATA_ENTRIES;
 298         }
 299
 300         (start, reso, list)
 301     }
 302
 303     /// Create instance from raw data, testing data len and magic number
 304     pub fn from_raw(mut raw: &[u8]) -> Result<Self, std::io::Error> {
 305         let expected_len = std::mem::size_of::<RRD>();
 306         if raw.len() != expected_len {
 307             let msg = format!("wrong data size ({} != {})", raw.len(), expected_len);
 308             return Err(std::io::Error::new(std::io::ErrorKind::Other, msg));
 309         }
 310
 311         let mut rrd: RRD = unsafe { std::mem::zeroed() };
 312         unsafe {
 313             let rrd_slice = std::slice::from_raw_parts_mut(&mut rrd as *mut _ as *mut u8, expected_len);
 314             raw.read_exact(rrd_slice)?;
 315         }
 316
 317         if rrd.magic != PROXMOX_RRD_MAGIC_1_0 {
 318             let msg = "wrong magic number".to_string();
 319             return Err(std::io::Error::new(std::io::ErrorKind::Other, msg));
 320         }
 321
 322         Ok(rrd)
 323     }
 324
 325     /// Load data from a file
 326     pub fn load(path: &Path) -> Result<Self, std::io::Error> {
 327         let raw = std::fs::read(path)?;
 328         Self::from_raw(&raw)
 329     }
 330
 331     /// Store data into a file (atomic replace file)
 332     pub fn save(&self, filename: &Path, options: CreateOptions) -> Result<(), Error> {
 333         let rrd_slice = unsafe {
 334             std::slice::from_raw_parts(self as *const _ as *const u8, std::mem::size_of::<RRD>())
 335         };
 336         replace_file(filename, rrd_slice, options)
 337     }
 338
 339     /// Update the value (in memory)
 340     ///
 341     /// Note: This does not call [Self::save].
 342     pub fn update(&mut self, time: f64, value: f64) {
 343
 344         let mut log_error = true;
 345
 346         let mut update_rra = |rra: &mut RRA| {
 347             if let Err(err) = rra.update(time, value) {
 348                 if log_error {
 349                     log::error!("rrd update failed: {}", err);
 350                     // we only log the first error, because it is very
 351                     // likely other calls produce the same error
 352                     log_error = false;
 353                 }
 354             }
 355         };
 356
 357         update_rra(&mut self.hour_avg);
 358         update_rra(&mut self.hour_max);
 359
 360         update_rra(&mut self.day_avg);
 361         update_rra(&mut self.day_max);
 362
 363         update_rra(&mut self.week_avg);
 364         update_rra(&mut self.week_max);
 365
 366         update_rra(&mut self.month_avg);
 367         update_rra(&mut self.month_max);
 368
 369         update_rra(&mut self.year_avg);
 370         update_rra(&mut self.year_max);
 371     }
 372 }