src/bin/proxmox-backup-proxy.rs

   1 use std::sync::Arc;
   2 use std::path::{Path, PathBuf};
   3
   4 use anyhow::{bail, format_err, Error};
   5 use futures::*;
   6 use hyper;
   7 use openssl::ssl::{SslMethod, SslAcceptor, SslFiletype};
   8
   9 use proxmox::try_block;
  10 use proxmox::api::RpcEnvironmentType;
  11
  12 use proxmox_backup::api2::types::Userid;
  13 use proxmox_backup::configdir;
  14 use proxmox_backup::buildcfg;
  15 use proxmox_backup::server;
  16 use proxmox_backup::tools::daemon;
  17 use proxmox_backup::server::{ApiConfig, rest::*};
  18 use proxmox_backup::auth_helpers::*;
  19 use proxmox_backup::tools::disks::{ DiskManage, zfs_pool_stats };
  20
  21 use proxmox_backup::api2::pull::do_sync_job;
  22
  23 fn main() -> Result<(), Error> {
  24     proxmox_backup::tools::setup_safe_path_env();
  25
  26     let backup_uid = proxmox_backup::backup::backup_user()?.uid;
  27     let backup_gid = proxmox_backup::backup::backup_group()?.gid;
  28     let running_uid = nix::unistd::Uid::effective();
  29     let running_gid = nix::unistd::Gid::effective();
  30
  31     if running_uid != backup_uid || running_gid != backup_gid {
  32         bail!("proxy not running as backup user or group (got uid {} gid {})", running_uid, running_gid);
  33     }
  34
  35     proxmox_backup::tools::runtime::main(run())
  36 }
  37
  38 async fn run() -> Result<(), Error> {
  39     if let Err(err) = syslog::init(
  40         syslog::Facility::LOG_DAEMON,
  41         log::LevelFilter::Info,
  42         Some("proxmox-backup-proxy")) {
  43         bail!("unable to inititialize syslog - {}", err);
  44     }
  45
  46     let _ = public_auth_key(); // load with lazy_static
  47     let _ = csrf_secret(); // load with lazy_static
  48
  49     let mut config = ApiConfig::new(
  50         buildcfg::JS_DIR, &proxmox_backup::api2::ROUTER, RpcEnvironmentType::PUBLIC)?;
  51
  52     config.add_alias("novnc", "/usr/share/novnc-pve");
  53     config.add_alias("extjs", "/usr/share/javascript/extjs");
  54     config.add_alias("fontawesome", "/usr/share/fonts-font-awesome");
  55     config.add_alias("xtermjs", "/usr/share/pve-xtermjs");
  56     config.add_alias("locale", "/usr/share/pbs-i18n");
  57     config.add_alias("widgettoolkit", "/usr/share/javascript/proxmox-widget-toolkit");
  58     config.add_alias("css", "/usr/share/javascript/proxmox-backup/css");
  59     config.add_alias("docs", "/usr/share/doc/proxmox-backup/html");
  60
  61     let mut indexpath = PathBuf::from(buildcfg::JS_DIR);
  62     indexpath.push("index.hbs");
  63     config.register_template("index", &indexpath)?;
  64     config.register_template("console", "/usr/share/pve-xtermjs/index.html.hbs")?;
  65
  66     let rest_server = RestServer::new(config);
  67
  68     //openssl req -x509 -newkey rsa:4096 -keyout /etc/proxmox-backup/proxy.key -out /etc/proxmox-backup/proxy.pem -nodes
  69     let key_path = configdir!("/proxy.key");
  70     let cert_path = configdir!("/proxy.pem");
  71
  72     let mut acceptor = SslAcceptor::mozilla_intermediate(SslMethod::tls()).unwrap();
  73     acceptor.set_private_key_file(key_path, SslFiletype::PEM)
  74         .map_err(|err| format_err!("unable to read proxy key {} - {}", key_path, err))?;
  75     acceptor.set_certificate_chain_file(cert_path)
  76         .map_err(|err| format_err!("unable to read proxy cert {} - {}", cert_path, err))?;
  77     acceptor.check_private_key().unwrap();
  78
  79     let acceptor = Arc::new(acceptor.build());
  80
  81     let server = daemon::create_daemon(
  82         ([0,0,0,0,0,0,0,0], 8007).into(),
  83         |listener, ready| {
  84             let connections = proxmox_backup::tools::async_io::StaticIncoming::from(listener)
  85                 .map_err(Error::from)
  86                 .try_filter_map(move |(sock, _addr)| {
  87                     let acceptor = Arc::clone(&acceptor);
  88                     async move {
  89                         sock.set_nodelay(true).unwrap();
  90                         Ok(tokio_openssl::accept(&acceptor, sock)
  91                             .await
  92                             .ok() // handshake errors aren't be fatal, so return None to filter
  93                         )
  94                     }
  95                 });
  96             let connections = proxmox_backup::tools::async_io::HyperAccept(connections);
  97
  98             Ok(ready
  99                 .and_then(|_| hyper::Server::builder(connections)
 100                     .serve(rest_server)
 101                     .with_graceful_shutdown(server::shutdown_future())
 102                     .map_err(Error::from)
 103                 )
 104                 .map_err(|err| eprintln!("server error: {}", err))
 105                 .map(|_| ())
 106             )
 107         },
 108     );
 109
 110     daemon::systemd_notify(daemon::SystemdNotify::Ready)?;
 111
 112     let init_result: Result<(), Error> = try_block!({
 113         server::create_task_control_socket()?;
 114         server::server_state_init()?;
 115         Ok(())
 116     });
 117
 118     if let Err(err) = init_result {
 119         bail!("unable to start daemon - {}", err);
 120     }
 121
 122     start_task_scheduler();
 123     start_stat_generator();
 124
 125     server.await?;
 126     log::info!("server shutting down, waiting for active workers to complete");
 127     proxmox_backup::server::last_worker_future().await?;
 128     log::info!("done - exit server");
 129
 130     Ok(())
 131 }
 132
 133 fn start_stat_generator() {
 134     let abort_future = server::shutdown_future();
 135     let future = Box::pin(run_stat_generator());
 136     let task = futures::future::select(future, abort_future);
 137     tokio::spawn(task.map(|_| ()));
 138 }
 139
 140 fn start_task_scheduler() {
 141     let abort_future = server::shutdown_future();
 142     let future = Box::pin(run_task_scheduler());
 143     let task = futures::future::select(future, abort_future);
 144     tokio::spawn(task.map(|_| ()));
 145 }
 146
 147 use std::time::{SystemTime, Instant, Duration, UNIX_EPOCH};
 148
 149 fn next_minute() -> Result<Instant, Error> {
 150     let now = SystemTime::now();
 151     let epoch_now = now.duration_since(UNIX_EPOCH)?;
 152     let epoch_next = Duration::from_secs((epoch_now.as_secs()/60  + 1)*60);
 153     Ok(Instant::now() + epoch_next - epoch_now)
 154 }
 155
 156 async fn run_task_scheduler() {
 157
 158     let mut count: usize = 0;
 159
 160     loop {
 161         count += 1;
 162
 163         let delay_target = match next_minute() {  // try to run very minute
 164             Ok(d) => d,
 165             Err(err) => {
 166                 eprintln!("task scheduler: compute next minute failed - {}", err);
 167                 tokio::time::delay_until(tokio::time::Instant::from_std(Instant::now() + Duration::from_secs(60))).await;
 168                 continue;
 169             }
 170         };
 171
 172         if count > 2 { // wait 1..2 minutes before starting
 173             match schedule_tasks().catch_unwind().await {
 174                 Err(panic) => {
 175                     match panic.downcast::<&str>() {
 176                         Ok(msg) => {
 177                             eprintln!("task scheduler panic: {}", msg);
 178                         }
 179                         Err(_) => {
 180                             eprintln!("task scheduler panic - unknown type");
 181                         }
 182                     }
 183                 }
 184                 Ok(Err(err)) => {
 185                     eprintln!("task scheduler failed - {:?}", err);
 186                 }
 187                 Ok(Ok(_)) => {}
 188             }
 189         }
 190
 191         tokio::time::delay_until(tokio::time::Instant::from_std(delay_target)).await;
 192     }
 193 }
 194
 195 async fn schedule_tasks() -> Result<(), Error> {
 196
 197     schedule_datastore_garbage_collection().await;
 198     schedule_datastore_prune().await;
 199     schedule_datastore_sync_jobs().await;
 200
 201     Ok(())
 202 }
 203
 204 fn lookup_last_worker(worker_type: &str, worker_id: &str) -> Result<Option<server::UPID>, Error> {
 205
 206     let list = proxmox_backup::server::read_task_list()?;
 207
 208     let mut last: Option<&server::UPID> = None;
 209
 210     for entry in list.iter() {
 211         if entry.upid.worker_type == worker_type {
 212             if let Some(ref id) = entry.upid.worker_id {
 213                 if id == worker_id {
 214                     match last {
 215                         Some(ref upid) => {
 216                             if upid.starttime < entry.upid.starttime {
 217                                 last = Some(&entry.upid)
 218                             }
 219                         }
 220                         None => {
 221                             last = Some(&entry.upid)
 222                         }
 223                     }
 224                 }
 225             }
 226         }
 227     }
 228
 229     Ok(last.cloned())
 230 }
 231
 232
 233 async fn schedule_datastore_garbage_collection() {
 234
 235     use proxmox_backup::backup::DataStore;
 236     use proxmox_backup::server::{UPID, WorkerTask};
 237     use proxmox_backup::config::datastore::{self, DataStoreConfig};
 238     use proxmox_backup::tools::systemd::time::{
 239         parse_calendar_event, compute_next_event};
 240
 241     let config = match datastore::config() {
 242         Err(err) => {
 243             eprintln!("unable to read datastore config - {}", err);
 244             return;
 245         }
 246         Ok((config, _digest)) => config,
 247     };
 248
 249     for (store, (_, store_config)) in config.sections {
 250         let datastore = match DataStore::lookup_datastore(&store) {
 251             Ok(datastore) => datastore,
 252             Err(err) => {
 253                 eprintln!("lookup_datastore failed - {}", err);
 254                 continue;
 255             }
 256         };
 257
 258         let store_config: DataStoreConfig = match serde_json::from_value(store_config) {
 259             Ok(c) => c,
 260             Err(err) => {
 261                 eprintln!("datastore config from_value failed - {}", err);
 262                 continue;
 263             }
 264         };
 265
 266         let event_str = match store_config.gc_schedule {
 267             Some(event_str) => event_str,
 268             None => continue,
 269         };
 270
 271         let event = match parse_calendar_event(&event_str) {
 272             Ok(event) => event,
 273             Err(err) => {
 274                 eprintln!("unable to parse schedule '{}' - {}", event_str, err);
 275                 continue;
 276             }
 277         };
 278
 279         if datastore.garbage_collection_running() { continue; }
 280
 281         let worker_type = "garbage_collection";
 282
 283         let stat = datastore.last_gc_status();
 284         let last = if let Some(upid_str) = stat.upid {
 285             match upid_str.parse::<UPID>() {
 286                 Ok(upid) => upid.starttime,
 287                 Err(err) => {
 288                     eprintln!("unable to parse upid '{}' - {}", upid_str, err);
 289                     continue;
 290                 }
 291             }
 292         } else {
 293             match lookup_last_worker(worker_type, &store) {
 294                 Ok(Some(upid)) => upid.starttime,
 295                 Ok(None) => 0,
 296                 Err(err) => {
 297                     eprintln!("lookup_last_job_start failed: {}", err);
 298                     continue;
 299                 }
 300             }
 301         };
 302
 303         let next = match compute_next_event(&event, last, false) {
 304             Ok(Some(next)) => next,
 305             Ok(None) => continue,
 306             Err(err) => {
 307                 eprintln!("compute_next_event for '{}' failed - {}", event_str, err);
 308                 continue;
 309             }
 310         };
 311
 312         let now = proxmox::tools::time::epoch_i64();
 313
 314         if next > now  { continue; }
 315
 316         let store2 = store.clone();
 317
 318         if let Err(err) = WorkerTask::new_thread(
 319             worker_type,
 320             Some(store.clone()),
 321             Userid::backup_userid().clone(),
 322             false,
 323             move |worker| {
 324                 worker.log(format!("starting garbage collection on store {}", store));
 325                 worker.log(format!("task triggered by schedule '{}'", event_str));
 326                 datastore.garbage_collection(&worker)
 327             }
 328         ) {
 329             eprintln!("unable to start garbage collection on store {} - {}", store2, err);
 330         }
 331     }
 332 }
 333
 334 async fn schedule_datastore_prune() {
 335
 336     use proxmox_backup::backup::{
 337         PruneOptions, DataStore, BackupGroup, compute_prune_info};
 338     use proxmox_backup::server::{WorkerTask};
 339     use proxmox_backup::config::datastore::{self, DataStoreConfig};
 340     use proxmox_backup::tools::systemd::time::{
 341         parse_calendar_event, compute_next_event};
 342
 343     let config = match datastore::config() {
 344         Err(err) => {
 345             eprintln!("unable to read datastore config - {}", err);
 346             return;
 347         }
 348         Ok((config, _digest)) => config,
 349     };
 350
 351     for (store, (_, store_config)) in config.sections {
 352         let datastore = match DataStore::lookup_datastore(&store) {
 353             Ok(datastore) => datastore,
 354             Err(err) => {
 355                 eprintln!("lookup_datastore '{}' failed - {}", store, err);
 356                 continue;
 357             }
 358         };
 359
 360         let store_config: DataStoreConfig = match serde_json::from_value(store_config) {
 361             Ok(c) => c,
 362             Err(err) => {
 363                 eprintln!("datastore '{}' config from_value failed - {}", store, err);
 364                 continue;
 365             }
 366         };
 367
 368         let event_str = match store_config.prune_schedule {
 369             Some(event_str) => event_str,
 370             None => continue,
 371         };
 372
 373         let prune_options = PruneOptions {
 374             keep_last: store_config.keep_last,
 375             keep_hourly: store_config.keep_hourly,
 376             keep_daily:  store_config.keep_daily,
 377             keep_weekly: store_config.keep_weekly,
 378             keep_monthly: store_config.keep_monthly,
 379             keep_yearly: store_config.keep_yearly,
 380         };
 381
 382         if !prune_options.keeps_something() { // no prune settings - keep all
 383             continue;
 384         }
 385
 386         let event = match parse_calendar_event(&event_str) {
 387             Ok(event) => event,
 388             Err(err) => {
 389                 eprintln!("unable to parse schedule '{}' - {}", event_str, err);
 390                 continue;
 391             }
 392         };
 393
 394         let worker_type = "prune";
 395
 396         let last = match lookup_last_worker(worker_type, &store) {
 397             Ok(Some(upid)) => {
 398                 if proxmox_backup::server::worker_is_active_local(&upid) {
 399                     continue;
 400                 }
 401                 upid.starttime
 402             }
 403             Ok(None) => 0,
 404             Err(err) => {
 405                 eprintln!("lookup_last_job_start failed: {}", err);
 406                 continue;
 407             }
 408         };
 409
 410         let next = match compute_next_event(&event, last, false) {
 411             Ok(Some(next)) => next,
 412             Ok(None) => continue,
 413             Err(err) => {
 414                 eprintln!("compute_next_event for '{}' failed - {}", event_str, err);
 415                 continue;
 416             }
 417         };
 418
 419         let now = proxmox::tools::time::epoch_i64();
 420
 421         if next > now  { continue; }
 422
 423         let store2 = store.clone();
 424
 425         if let Err(err) = WorkerTask::new_thread(
 426             worker_type,
 427             Some(store.clone()),
 428             Userid::backup_userid().clone(),
 429             false,
 430             move |worker| {
 431                 worker.log(format!("Starting datastore prune on store \"{}\"", store));
 432                 worker.log(format!("task triggered by schedule '{}'", event_str));
 433                 worker.log(format!("retention options: {}", prune_options.cli_options_string()));
 434
 435                 let base_path = datastore.base_path();
 436
 437                 let groups = BackupGroup::list_groups(&base_path)?;
 438                 for group in groups {
 439                     let list = group.list_backups(&base_path)?;
 440                     let mut prune_info = compute_prune_info(list, &prune_options)?;
 441                     prune_info.reverse(); // delete older snapshots first
 442
 443                     worker.log(format!("Starting prune on store \"{}\" group \"{}/{}\"",
 444                                        store, group.backup_type(), group.backup_id()));
 445
 446                     for (info, keep) in prune_info {
 447                         worker.log(format!(
 448                             "{} {}/{}/{}",
 449                             if keep { "keep" } else { "remove" },
 450                             group.backup_type(), group.backup_id(),
 451                             info.backup_dir.backup_time_string()));
 452                         if !keep {
 453                             datastore.remove_backup_dir(&info.backup_dir, true)?;
 454                         }
 455                     }
 456                 }
 457
 458                 Ok(())
 459             }
 460         ) {
 461             eprintln!("unable to start datastore prune on store {} - {}", store2, err);
 462         }
 463     }
 464 }
 465
 466 async fn schedule_datastore_sync_jobs() {
 467
 468     use proxmox_backup::{
 469         config::{ sync::{self, SyncJobConfig}, jobstate::{self, Job} },
 470         tools::systemd::time::{ parse_calendar_event, compute_next_event },
 471     };
 472
 473     let config = match sync::config() {
 474         Err(err) => {
 475             eprintln!("unable to read sync job config - {}", err);
 476             return;
 477         }
 478         Ok((config, _digest)) => config,
 479     };
 480
 481     for (job_id, (_, job_config)) in config.sections {
 482         let job_config: SyncJobConfig = match serde_json::from_value(job_config) {
 483             Ok(c) => c,
 484             Err(err) => {
 485                 eprintln!("sync job config from_value failed - {}", err);
 486                 continue;
 487             }
 488         };
 489
 490         let event_str = match job_config.schedule {
 491             Some(ref event_str) => event_str.clone(),
 492             None => continue,
 493         };
 494
 495         let event = match parse_calendar_event(&event_str) {
 496             Ok(event) => event,
 497             Err(err) => {
 498                 eprintln!("unable to parse schedule '{}' - {}", event_str, err);
 499                 continue;
 500             }
 501         };
 502
 503         let worker_type = "syncjob";
 504
 505         let last = match jobstate::last_run_time(worker_type, &job_id) {
 506             Ok(time) => time,
 507             Err(err) => {
 508                 eprintln!("could not get last run time of {} {}: {}", worker_type, job_id, err);
 509                 continue;
 510             }
 511         };
 512
 513         let next = match compute_next_event(&event, last, false) {
 514             Ok(Some(next)) => next,
 515             Ok(None) => continue,
 516             Err(err) => {
 517                 eprintln!("compute_next_event for '{}' failed - {}", event_str, err);
 518                 continue;
 519             }
 520         };
 521
 522         let now = proxmox::tools::time::epoch_i64();
 523
 524         if next > now  { continue; }
 525
 526         let job = match Job::new(worker_type, &job_id) {
 527             Ok(job) => job,
 528             Err(_) => continue, // could not get lock
 529         };
 530
 531         let userid = Userid::backup_userid().clone();
 532
 533         if let Err(err) = do_sync_job(job, job_config, &userid, Some(event_str)) {
 534             eprintln!("unable to start datastore sync job {} - {}", &job_id, err);
 535         }
 536     }
 537 }
 538
 539 async fn run_stat_generator() {
 540
 541     let mut count = 0;
 542     loop {
 543         count += 1;
 544         let save = if count >= 6 { count = 0; true } else { false };
 545
 546         let delay_target = Instant::now() +  Duration::from_secs(10);
 547
 548         generate_host_stats(save).await;
 549
 550         tokio::time::delay_until(tokio::time::Instant::from_std(delay_target)).await;
 551
 552      }
 553
 554 }
 555
 556 fn rrd_update_gauge(name: &str, value: f64, save: bool) {
 557     use proxmox_backup::rrd;
 558     if let Err(err) = rrd::update_value(name, value, rrd::DST::Gauge, save) {
 559         eprintln!("rrd::update_value '{}' failed - {}", name, err);
 560     }
 561 }
 562
 563 fn rrd_update_derive(name: &str, value: f64, save: bool) {
 564     use proxmox_backup::rrd;
 565     if let Err(err) = rrd::update_value(name, value, rrd::DST::Derive, save) {
 566         eprintln!("rrd::update_value '{}' failed - {}", name, err);
 567     }
 568 }
 569
 570 async fn generate_host_stats(save: bool) {
 571     use proxmox::sys::linux::procfs::{
 572         read_meminfo, read_proc_stat, read_proc_net_dev, read_loadavg};
 573     use proxmox_backup::config::datastore;
 574
 575
 576     proxmox_backup::tools::runtime::block_in_place(move || {
 577
 578         match read_proc_stat() {
 579             Ok(stat) => {
 580                 rrd_update_gauge("host/cpu", stat.cpu, save);
 581                 rrd_update_gauge("host/iowait", stat.iowait_percent, save);
 582             }
 583             Err(err) => {
 584                 eprintln!("read_proc_stat failed - {}", err);
 585             }
 586         }
 587
 588         match read_meminfo() {
 589             Ok(meminfo) => {
 590                 rrd_update_gauge("host/memtotal", meminfo.memtotal as f64, save);
 591                 rrd_update_gauge("host/memused", meminfo.memused as f64, save);
 592                 rrd_update_gauge("host/swaptotal", meminfo.swaptotal as f64, save);
 593                 rrd_update_gauge("host/swapused", meminfo.swapused as f64, save);
 594             }
 595             Err(err) => {
 596                 eprintln!("read_meminfo failed - {}", err);
 597             }
 598         }
 599
 600         match read_proc_net_dev() {
 601             Ok(netdev) => {
 602                 use proxmox_backup::config::network::is_physical_nic;
 603                 let mut netin = 0;
 604                 let mut netout = 0;
 605                 for item in netdev {
 606                     if !is_physical_nic(&item.device) { continue; }
 607                     netin += item.receive;
 608                     netout += item.send;
 609                 }
 610                 rrd_update_derive("host/netin", netin as f64, save);
 611                 rrd_update_derive("host/netout", netout as f64, save);
 612             }
 613             Err(err) => {
 614                 eprintln!("read_prox_net_dev failed - {}", err);
 615             }
 616         }
 617
 618         match read_loadavg() {
 619             Ok(loadavg) => {
 620                 rrd_update_gauge("host/loadavg", loadavg.0 as f64, save);
 621             }
 622             Err(err) => {
 623                 eprintln!("read_loadavg failed - {}", err);
 624             }
 625         }
 626
 627         let disk_manager = DiskManage::new();
 628
 629         gather_disk_stats(disk_manager.clone(), Path::new("/"), "host", save);
 630
 631         match datastore::config() {
 632             Ok((config, _)) => {
 633                 let datastore_list: Vec<datastore::DataStoreConfig> =
 634                     config.convert_to_typed_array("datastore").unwrap_or(Vec::new());
 635
 636                 for config in datastore_list {
 637
 638                     let rrd_prefix = format!("datastore/{}", config.name);
 639                     let path = std::path::Path::new(&config.path);
 640                     gather_disk_stats(disk_manager.clone(), path, &rrd_prefix, save);
 641                 }
 642             }
 643             Err(err) => {
 644                 eprintln!("read datastore config failed - {}", err);
 645             }
 646         }
 647
 648     });
 649 }
 650
 651 fn gather_disk_stats(disk_manager: Arc<DiskManage>, path: &Path, rrd_prefix: &str, save: bool) {
 652
 653     match proxmox_backup::tools::disks::disk_usage(path) {
 654         Ok(status) => {
 655             let rrd_key = format!("{}/total", rrd_prefix);
 656             rrd_update_gauge(&rrd_key, status.total as f64, save);
 657             let rrd_key = format!("{}/used", rrd_prefix);
 658             rrd_update_gauge(&rrd_key, status.used as f64, save);
 659         }
 660         Err(err) => {
 661             eprintln!("read disk_usage on {:?} failed - {}", path, err);
 662         }
 663     }
 664
 665     match disk_manager.find_mounted_device(path) {
 666         Ok(None) => {},
 667         Ok(Some((fs_type, device, source))) => {
 668             let mut device_stat = None;
 669             match fs_type.as_str() {
 670                 "zfs" => {
 671                     if let Some(pool) = source {
 672                         match zfs_pool_stats(&pool) {
 673                             Ok(stat) => device_stat = stat,
 674                             Err(err) => eprintln!("zfs_pool_stats({:?}) failed - {}", pool, err),
 675                         }
 676                     }
 677                 }
 678                 _ => {
 679                     if let Ok(disk) = disk_manager.clone().disk_by_dev_num(device.into_dev_t()) {
 680                         match disk.read_stat() {
 681                             Ok(stat) => device_stat = stat,
 682                             Err(err) => eprintln!("disk.read_stat {:?} failed - {}", path, err),
 683                         }
 684                     }
 685                 }
 686             }
 687             if let Some(stat) = device_stat {
 688                 let rrd_key = format!("{}/read_ios", rrd_prefix);
 689                 rrd_update_derive(&rrd_key, stat.read_ios as f64, save);
 690                 let rrd_key = format!("{}/read_bytes", rrd_prefix);
 691                 rrd_update_derive(&rrd_key, (stat.read_sectors*512) as f64, save);
 692
 693                 let rrd_key = format!("{}/write_ios", rrd_prefix);
 694                 rrd_update_derive(&rrd_key, stat.write_ios as f64, save);
 695                 let rrd_key = format!("{}/write_bytes", rrd_prefix);
 696                 rrd_update_derive(&rrd_key, (stat.write_sectors*512) as f64, save);
 697
 698                 let rrd_key = format!("{}/io_ticks", rrd_prefix);
 699                 rrd_update_derive(&rrd_key, (stat.io_ticks as f64)/1000.0, save);
 700             }
 701         }
 702         Err(err) => {
 703             eprintln!("find_mounted_device failed - {}", err);
 704         }
 705     }
 706 }