src/bin/proxmox-backup-proxy.rs

   1 use std::sync::Arc;
   2 use std::path::{Path, PathBuf};
   3
   4 use anyhow::{bail, format_err, Error};
   5 use futures::*;
   6 use hyper;
   7 use openssl::ssl::{SslMethod, SslAcceptor, SslFiletype};
   8
   9 use proxmox::try_block;
  10 use proxmox::api::RpcEnvironmentType;
  11
  12 use proxmox_backup::api2::types::Userid;
  13 use proxmox_backup::configdir;
  14 use proxmox_backup::buildcfg;
  15 use proxmox_backup::server;
  16 use proxmox_backup::tools::{daemon, epoch_now, epoch_now_u64};
  17 use proxmox_backup::server::{ApiConfig, rest::*};
  18 use proxmox_backup::auth_helpers::*;
  19 use proxmox_backup::tools::disks::{ DiskManage, zfs_pool_stats };
  20
  21 use proxmox_backup::api2::pull::do_sync_job;
  22
  23 fn main() {
  24     proxmox_backup::tools::setup_safe_path_env();
  25
  26     if let Err(err) = proxmox_backup::tools::runtime::main(run()) {
  27         eprintln!("Error: {}", err);
  28         std::process::exit(-1);
  29     }
  30 }
  31
  32 async fn run() -> Result<(), Error> {
  33     if let Err(err) = syslog::init(
  34         syslog::Facility::LOG_DAEMON,
  35         log::LevelFilter::Info,
  36         Some("proxmox-backup-proxy")) {
  37         bail!("unable to inititialize syslog - {}", err);
  38     }
  39
  40     let _ = public_auth_key(); // load with lazy_static
  41     let _ = csrf_secret(); // load with lazy_static
  42
  43     let mut config = ApiConfig::new(
  44         buildcfg::JS_DIR, &proxmox_backup::api2::ROUTER, RpcEnvironmentType::PUBLIC)?;
  45
  46     // add default dirs which includes jquery and bootstrap
  47     // my $base = '/usr/share/libpve-http-server-perl';
  48     // add_dirs($self->{dirs}, '/css/' => "$base/css/");
  49     // add_dirs($self->{dirs}, '/js/' => "$base/js/");
  50     // add_dirs($self->{dirs}, '/fonts/' => "$base/fonts/");
  51     config.add_alias("novnc", "/usr/share/novnc-pve");
  52     config.add_alias("extjs", "/usr/share/javascript/extjs");
  53     config.add_alias("fontawesome", "/usr/share/fonts-font-awesome");
  54     config.add_alias("xtermjs", "/usr/share/pve-xtermjs");
  55     config.add_alias("widgettoolkit", "/usr/share/javascript/proxmox-widget-toolkit");
  56     config.add_alias("css", "/usr/share/javascript/proxmox-backup/css");
  57     config.add_alias("docs", "/usr/share/doc/proxmox-backup/html");
  58
  59     let mut indexpath = PathBuf::from(buildcfg::JS_DIR);
  60     indexpath.push("index.hbs");
  61     config.register_template("index", &indexpath)?;
  62     config.register_template("console", "/usr/share/pve-xtermjs/index.html.hbs")?;
  63
  64     let rest_server = RestServer::new(config);
  65
  66     //openssl req -x509 -newkey rsa:4096 -keyout /etc/proxmox-backup/proxy.key -out /etc/proxmox-backup/proxy.pem -nodes
  67     let key_path = configdir!("/proxy.key");
  68     let cert_path = configdir!("/proxy.pem");
  69
  70     let mut acceptor = SslAcceptor::mozilla_intermediate(SslMethod::tls()).unwrap();
  71     acceptor.set_private_key_file(key_path, SslFiletype::PEM)
  72         .map_err(|err| format_err!("unable to read proxy key {} - {}", key_path, err))?;
  73     acceptor.set_certificate_chain_file(cert_path)
  74         .map_err(|err| format_err!("unable to read proxy cert {} - {}", cert_path, err))?;
  75     acceptor.check_private_key().unwrap();
  76
  77     let acceptor = Arc::new(acceptor.build());
  78
  79     let server = daemon::create_daemon(
  80         ([0,0,0,0,0,0,0,0], 8007).into(),
  81         |listener, ready| {
  82             let connections = proxmox_backup::tools::async_io::StaticIncoming::from(listener)
  83                 .map_err(Error::from)
  84                 .try_filter_map(move |(sock, _addr)| {
  85                     let acceptor = Arc::clone(&acceptor);
  86                     async move {
  87                         sock.set_nodelay(true).unwrap();
  88                         sock.set_send_buffer_size(1024*1024).unwrap();
  89                         sock.set_recv_buffer_size(1024*1024).unwrap();
  90                         Ok(tokio_openssl::accept(&acceptor, sock)
  91                             .await
  92                             .ok() // handshake errors aren't be fatal, so return None to filter
  93                         )
  94                     }
  95                 });
  96             let connections = proxmox_backup::tools::async_io::HyperAccept(connections);
  97
  98             Ok(ready
  99                 .and_then(|_| hyper::Server::builder(connections)
 100                     .serve(rest_server)
 101                     .with_graceful_shutdown(server::shutdown_future())
 102                     .map_err(Error::from)
 103                 )
 104                 .map_err(|err| eprintln!("server error: {}", err))
 105                 .map(|_| ())
 106             )
 107         },
 108     );
 109
 110     daemon::systemd_notify(daemon::SystemdNotify::Ready)?;
 111
 112     let init_result: Result<(), Error> = try_block!({
 113         server::create_task_control_socket()?;
 114         server::server_state_init()?;
 115         Ok(())
 116     });
 117
 118     if let Err(err) = init_result {
 119         bail!("unable to start daemon - {}", err);
 120     }
 121
 122     start_task_scheduler();
 123     start_stat_generator();
 124
 125     server.await?;
 126     log::info!("server shutting down, waiting for active workers to complete");
 127     proxmox_backup::server::last_worker_future().await?;
 128     log::info!("done - exit server");
 129
 130     Ok(())
 131 }
 132
 133 fn start_stat_generator() {
 134     let abort_future = server::shutdown_future();
 135     let future = Box::pin(run_stat_generator());
 136     let task = futures::future::select(future, abort_future);
 137     tokio::spawn(task.map(|_| ()));
 138 }
 139
 140 fn start_task_scheduler() {
 141     let abort_future = server::shutdown_future();
 142     let future = Box::pin(run_task_scheduler());
 143     let task = futures::future::select(future, abort_future);
 144     tokio::spawn(task.map(|_| ()));
 145 }
 146
 147 use std::time:: {Instant, Duration};
 148
 149 fn next_minute() -> Result<Instant, Error> {
 150     let epoch_now = epoch_now()?;
 151     let epoch_next = Duration::from_secs((epoch_now.as_secs()/60 + 1)*60);
 152     Ok(Instant::now() + epoch_next - epoch_now)
 153 }
 154
 155 async fn run_task_scheduler() {
 156
 157     let mut count: usize = 0;
 158
 159     loop {
 160         count += 1;
 161
 162         let delay_target = match next_minute() {  // try to run very minute
 163             Ok(d) => d,
 164             Err(err) => {
 165                 eprintln!("task scheduler: compute next minute failed - {}", err);
 166                 tokio::time::delay_until(tokio::time::Instant::from_std(Instant::now() + Duration::from_secs(60))).await;
 167                 continue;
 168             }
 169         };
 170
 171         if count > 2 { // wait 1..2 minutes before starting
 172             match schedule_tasks().catch_unwind().await {
 173                 Err(panic) => {
 174                     match panic.downcast::<&str>() {
 175                         Ok(msg) => {
 176                             eprintln!("task scheduler panic: {}", msg);
 177                         }
 178                         Err(_) => {
 179                             eprintln!("task scheduler panic - unknown type");
 180                         }
 181                     }
 182                 }
 183                 Ok(Err(err)) => {
 184                     eprintln!("task scheduler failed - {:?}", err);
 185                 }
 186                 Ok(Ok(_)) => {}
 187             }
 188         }
 189
 190         tokio::time::delay_until(tokio::time::Instant::from_std(delay_target)).await;
 191     }
 192 }
 193
 194 async fn schedule_tasks() -> Result<(), Error> {
 195
 196     schedule_datastore_garbage_collection().await;
 197     schedule_datastore_prune().await;
 198     schedule_datastore_sync_jobs().await;
 199
 200     Ok(())
 201 }
 202
 203 fn lookup_last_worker(worker_type: &str, worker_id: &str) -> Result<Option<server::UPID>, Error> {
 204
 205     let list = proxmox_backup::server::read_task_list()?;
 206
 207     let mut last: Option<&server::UPID> = None;
 208
 209     for entry in list.iter() {
 210         if entry.upid.worker_type == worker_type {
 211             if let Some(ref id) = entry.upid.worker_id {
 212                 if id == worker_id {
 213                     match last {
 214                         Some(ref upid) => {
 215                             if upid.starttime < entry.upid.starttime {
 216                                 last = Some(&entry.upid)
 217                             }
 218                         }
 219                         None => {
 220                             last = Some(&entry.upid)
 221                         }
 222                     }
 223                 }
 224             }
 225         }
 226     }
 227
 228     Ok(last.cloned())
 229 }
 230
 231
 232 async fn schedule_datastore_garbage_collection() {
 233
 234     use proxmox_backup::backup::DataStore;
 235     use proxmox_backup::server::{UPID, WorkerTask};
 236     use proxmox_backup::config::datastore::{self, DataStoreConfig};
 237     use proxmox_backup::tools::systemd::time::{
 238         parse_calendar_event, compute_next_event};
 239
 240     let config = match datastore::config() {
 241         Err(err) => {
 242             eprintln!("unable to read datastore config - {}", err);
 243             return;
 244         }
 245         Ok((config, _digest)) => config,
 246     };
 247
 248     for (store, (_, store_config)) in config.sections {
 249         let datastore = match DataStore::lookup_datastore(&store) {
 250             Ok(datastore) => datastore,
 251             Err(err) => {
 252                 eprintln!("lookup_datastore failed - {}", err);
 253                 continue;
 254             }
 255         };
 256
 257         let store_config: DataStoreConfig = match serde_json::from_value(store_config) {
 258             Ok(c) => c,
 259             Err(err) => {
 260                 eprintln!("datastore config from_value failed - {}", err);
 261                 continue;
 262             }
 263         };
 264
 265         let event_str = match store_config.gc_schedule {
 266             Some(event_str) => event_str,
 267             None => continue,
 268         };
 269
 270         let event = match parse_calendar_event(&event_str) {
 271             Ok(event) => event,
 272             Err(err) => {
 273                 eprintln!("unable to parse schedule '{}' - {}", event_str, err);
 274                 continue;
 275             }
 276         };
 277
 278         if datastore.garbage_collection_running() { continue; }
 279
 280         let worker_type = "garbage_collection";
 281
 282         let stat = datastore.last_gc_status();
 283         let last = if let Some(upid_str) = stat.upid {
 284             match upid_str.parse::<UPID>() {
 285                 Ok(upid) => upid.starttime,
 286                 Err(err) => {
 287                     eprintln!("unable to parse upid '{}' - {}", upid_str, err);
 288                     continue;
 289                 }
 290             }
 291         } else {
 292             match lookup_last_worker(worker_type, &store) {
 293                 Ok(Some(upid)) => upid.starttime,
 294                 Ok(None) => 0,
 295                 Err(err) => {
 296                     eprintln!("lookup_last_job_start failed: {}", err);
 297                     continue;
 298                 }
 299             }
 300         };
 301
 302         let next = match compute_next_event(&event, last, false) {
 303             Ok(next) => next,
 304             Err(err) => {
 305                 eprintln!("compute_next_event for '{}' failed - {}", event_str, err);
 306                 continue;
 307             }
 308         };
 309
 310         let now = match epoch_now_u64() {
 311             Ok(epoch_now) => epoch_now as i64,
 312             Err(err) => {
 313                 eprintln!("query system time failed - {}", err);
 314                 continue;
 315             }
 316         };
 317         if next > now  { continue; }
 318
 319         let store2 = store.clone();
 320
 321         if let Err(err) = WorkerTask::new_thread(
 322             worker_type,
 323             Some(store.clone()),
 324             Userid::backup_userid().clone(),
 325             false,
 326             move |worker| {
 327                 worker.log(format!("starting garbage collection on store {}", store));
 328                 worker.log(format!("task triggered by schedule '{}'", event_str));
 329                 datastore.garbage_collection(&worker)
 330             }
 331         ) {
 332             eprintln!("unable to start garbage collection on store {} - {}", store2, err);
 333         }
 334     }
 335 }
 336
 337 async fn schedule_datastore_prune() {
 338
 339     use proxmox_backup::backup::{
 340         PruneOptions, DataStore, BackupGroup, BackupDir, compute_prune_info};
 341     use proxmox_backup::server::{WorkerTask};
 342     use proxmox_backup::config::datastore::{self, DataStoreConfig};
 343     use proxmox_backup::tools::systemd::time::{
 344         parse_calendar_event, compute_next_event};
 345
 346     let config = match datastore::config() {
 347         Err(err) => {
 348             eprintln!("unable to read datastore config - {}", err);
 349             return;
 350         }
 351         Ok((config, _digest)) => config,
 352     };
 353
 354     for (store, (_, store_config)) in config.sections {
 355         let datastore = match DataStore::lookup_datastore(&store) {
 356             Ok(datastore) => datastore,
 357             Err(err) => {
 358                 eprintln!("lookup_datastore '{}' failed - {}", store, err);
 359                 continue;
 360             }
 361         };
 362
 363         let store_config: DataStoreConfig = match serde_json::from_value(store_config) {
 364             Ok(c) => c,
 365             Err(err) => {
 366                 eprintln!("datastore '{}' config from_value failed - {}", store, err);
 367                 continue;
 368             }
 369         };
 370
 371         let event_str = match store_config.prune_schedule {
 372             Some(event_str) => event_str,
 373             None => continue,
 374         };
 375
 376         let prune_options = PruneOptions {
 377             keep_last: store_config.keep_last,
 378             keep_hourly: store_config.keep_hourly,
 379             keep_daily:  store_config.keep_daily,
 380             keep_weekly: store_config.keep_weekly,
 381             keep_monthly: store_config.keep_monthly,
 382             keep_yearly: store_config.keep_yearly,
 383         };
 384
 385         if !prune_options.keeps_something() { // no prune settings - keep all
 386             continue;
 387         }
 388
 389         let event = match parse_calendar_event(&event_str) {
 390             Ok(event) => event,
 391             Err(err) => {
 392                 eprintln!("unable to parse schedule '{}' - {}", event_str, err);
 393                 continue;
 394             }
 395         };
 396
 397         let worker_type = "prune";
 398
 399         let last = match lookup_last_worker(worker_type, &store) {
 400             Ok(Some(upid)) => {
 401                 if proxmox_backup::server::worker_is_active_local(&upid) {
 402                     continue;
 403                 }
 404                 upid.starttime
 405             }
 406             Ok(None) => 0,
 407             Err(err) => {
 408                 eprintln!("lookup_last_job_start failed: {}", err);
 409                 continue;
 410             }
 411         };
 412
 413         let next = match compute_next_event(&event, last, false) {
 414             Ok(next) => next,
 415             Err(err) => {
 416                 eprintln!("compute_next_event for '{}' failed - {}", event_str, err);
 417                 continue;
 418             }
 419         };
 420
 421         let now = match epoch_now_u64() {
 422             Ok(epoch_now) => epoch_now as i64,
 423             Err(err) => {
 424                 eprintln!("query system time failed - {}", err);
 425                 continue;
 426             }
 427         };
 428         if next > now  { continue; }
 429
 430         let store2 = store.clone();
 431
 432         if let Err(err) = WorkerTask::new_thread(
 433             worker_type,
 434             Some(store.clone()),
 435             Userid::backup_userid().clone(),
 436             false,
 437             move |worker| {
 438                 worker.log(format!("Starting datastore prune on store \"{}\"", store));
 439                 worker.log(format!("task triggered by schedule '{}'", event_str));
 440                 worker.log(format!("retention options: {}", prune_options.cli_options_string()));
 441
 442                 let base_path = datastore.base_path();
 443
 444                 let groups = BackupGroup::list_groups(&base_path)?;
 445                 for group in groups {
 446                     let list = group.list_backups(&base_path)?;
 447                     let mut prune_info = compute_prune_info(list, &prune_options)?;
 448                     prune_info.reverse(); // delete older snapshots first
 449
 450                     worker.log(format!("Starting prune on store \"{}\" group \"{}/{}\"",
 451                                        store, group.backup_type(), group.backup_id()));
 452
 453                     for (info, keep) in prune_info {
 454                         worker.log(format!(
 455                             "{} {}/{}/{}",
 456                             if keep { "keep" } else { "remove" },
 457                             group.backup_type(), group.backup_id(),
 458                             BackupDir::backup_time_to_string(info.backup_dir.backup_time())));
 459
 460                         if !keep {
 461                             datastore.remove_backup_dir(&info.backup_dir, true)?;
 462                         }
 463                     }
 464                 }
 465
 466                 Ok(())
 467             }
 468         ) {
 469             eprintln!("unable to start datastore prune on store {} - {}", store2, err);
 470         }
 471     }
 472 }
 473
 474 async fn schedule_datastore_sync_jobs() {
 475
 476     use proxmox_backup::{
 477         config::{ sync::{self, SyncJobConfig}, jobstate::{self, Job} },
 478         tools::systemd::time::{ parse_calendar_event, compute_next_event },
 479     };
 480
 481     let config = match sync::config() {
 482         Err(err) => {
 483             eprintln!("unable to read sync job config - {}", err);
 484             return;
 485         }
 486         Ok((config, _digest)) => config,
 487     };
 488
 489     for (job_id, (_, job_config)) in config.sections {
 490         let job_config: SyncJobConfig = match serde_json::from_value(job_config) {
 491             Ok(c) => c,
 492             Err(err) => {
 493                 eprintln!("sync job config from_value failed - {}", err);
 494                 continue;
 495             }
 496         };
 497
 498         let event_str = match job_config.schedule {
 499             Some(ref event_str) => event_str.clone(),
 500             None => continue,
 501         };
 502
 503         let event = match parse_calendar_event(&event_str) {
 504             Ok(event) => event,
 505             Err(err) => {
 506                 eprintln!("unable to parse schedule '{}' - {}", event_str, err);
 507                 continue;
 508             }
 509         };
 510
 511         let worker_type = "syncjob";
 512
 513         let last = match jobstate::last_run_time(worker_type, &job_id) {
 514             Ok(time) => time,
 515             Err(err) => {
 516                 eprintln!("could not get last run time of {} {}: {}", worker_type, job_id, err);
 517                 continue;
 518             }
 519         };
 520
 521         let next = match compute_next_event(&event, last, false) {
 522             Ok(next) => next,
 523             Err(err) => {
 524                 eprintln!("compute_next_event for '{}' failed - {}", event_str, err);
 525                 continue;
 526             }
 527         };
 528
 529         let now = match epoch_now_u64() {
 530             Ok(epoch_now) => epoch_now as i64,
 531             Err(err) => {
 532                 eprintln!("query system time failed - {}", err);
 533                 continue;
 534             }
 535         };
 536         if next > now  { continue; }
 537
 538         let job = match Job::new(worker_type, &job_id) {
 539             Ok(job) => job,
 540             Err(_) => continue, // could not get lock
 541         };
 542
 543         let userid = Userid::backup_userid().clone();
 544
 545         if let Err(err) = do_sync_job(job, job_config, &userid, Some(event_str)) {
 546             eprintln!("unable to start datastore sync job {} - {}", &job_id, err);
 547         }
 548     }
 549 }
 550
 551 async fn run_stat_generator() {
 552
 553     let mut count = 0;
 554     loop {
 555         count += 1;
 556         let save = if count >= 6 { count = 0; true } else { false };
 557
 558         let delay_target = Instant::now() +  Duration::from_secs(10);
 559
 560         generate_host_stats(save).await;
 561
 562         tokio::time::delay_until(tokio::time::Instant::from_std(delay_target)).await;
 563
 564      }
 565
 566 }
 567
 568 fn rrd_update_gauge(name: &str, value: f64, save: bool) {
 569     use proxmox_backup::rrd;
 570     if let Err(err) = rrd::update_value(name, value, rrd::DST::Gauge, save) {
 571         eprintln!("rrd::update_value '{}' failed - {}", name, err);
 572     }
 573 }
 574
 575 fn rrd_update_derive(name: &str, value: f64, save: bool) {
 576     use proxmox_backup::rrd;
 577     if let Err(err) = rrd::update_value(name, value, rrd::DST::Derive, save) {
 578         eprintln!("rrd::update_value '{}' failed - {}", name, err);
 579     }
 580 }
 581
 582 async fn generate_host_stats(save: bool) {
 583     use proxmox::sys::linux::procfs::{
 584         read_meminfo, read_proc_stat, read_proc_net_dev, read_loadavg};
 585     use proxmox_backup::config::datastore;
 586
 587
 588     proxmox_backup::tools::runtime::block_in_place(move || {
 589
 590         match read_proc_stat() {
 591             Ok(stat) => {
 592                 rrd_update_gauge("host/cpu", stat.cpu, save);
 593                 rrd_update_gauge("host/iowait", stat.iowait_percent, save);
 594             }
 595             Err(err) => {
 596                 eprintln!("read_proc_stat failed - {}", err);
 597             }
 598         }
 599
 600         match read_meminfo() {
 601             Ok(meminfo) => {
 602                 rrd_update_gauge("host/memtotal", meminfo.memtotal as f64, save);
 603                 rrd_update_gauge("host/memused", meminfo.memused as f64, save);
 604                 rrd_update_gauge("host/swaptotal", meminfo.swaptotal as f64, save);
 605                 rrd_update_gauge("host/swapused", meminfo.swapused as f64, save);
 606             }
 607             Err(err) => {
 608                 eprintln!("read_meminfo failed - {}", err);
 609             }
 610         }
 611
 612         match read_proc_net_dev() {
 613             Ok(netdev) => {
 614                 use proxmox_backup::config::network::is_physical_nic;
 615                 let mut netin = 0;
 616                 let mut netout = 0;
 617                 for item in netdev {
 618                     if !is_physical_nic(&item.device) { continue; }
 619                     netin += item.receive;
 620                     netout += item.send;
 621                 }
 622                 rrd_update_derive("host/netin", netin as f64, save);
 623                 rrd_update_derive("host/netout", netout as f64, save);
 624             }
 625             Err(err) => {
 626                 eprintln!("read_prox_net_dev failed - {}", err);
 627             }
 628         }
 629
 630         match read_loadavg() {
 631             Ok(loadavg) => {
 632                 rrd_update_gauge("host/loadavg", loadavg.0 as f64, save);
 633             }
 634             Err(err) => {
 635                 eprintln!("read_loadavg failed - {}", err);
 636             }
 637         }
 638
 639         let disk_manager = DiskManage::new();
 640
 641         gather_disk_stats(disk_manager.clone(), Path::new("/"), "host", save);
 642
 643         match datastore::config() {
 644             Ok((config, _)) => {
 645                 let datastore_list: Vec<datastore::DataStoreConfig> =
 646                     config.convert_to_typed_array("datastore").unwrap_or(Vec::new());
 647
 648                 for config in datastore_list {
 649
 650                     let rrd_prefix = format!("datastore/{}", config.name);
 651                     let path = std::path::Path::new(&config.path);
 652                     gather_disk_stats(disk_manager.clone(), path, &rrd_prefix, save);
 653                 }
 654             }
 655             Err(err) => {
 656                 eprintln!("read datastore config failed - {}", err);
 657             }
 658         }
 659
 660     });
 661 }
 662
 663 fn gather_disk_stats(disk_manager: Arc<DiskManage>, path: &Path, rrd_prefix: &str, save: bool) {
 664
 665     match proxmox_backup::tools::disks::disk_usage(path) {
 666         Ok(status) => {
 667             let rrd_key = format!("{}/total", rrd_prefix);
 668             rrd_update_gauge(&rrd_key, status.total as f64, save);
 669             let rrd_key = format!("{}/used", rrd_prefix);
 670             rrd_update_gauge(&rrd_key, status.used as f64, save);
 671         }
 672         Err(err) => {
 673             eprintln!("read disk_usage on {:?} failed - {}", path, err);
 674         }
 675     }
 676
 677     match disk_manager.find_mounted_device(path) {
 678         Ok(None) => {},
 679         Ok(Some((fs_type, device, source))) => {
 680             let mut device_stat = None;
 681             match fs_type.as_str() {
 682                 "zfs" => {
 683                     if let Some(pool) = source {
 684                         match zfs_pool_stats(&pool) {
 685                             Ok(stat) => device_stat = stat,
 686                             Err(err) => eprintln!("zfs_pool_stats({:?}) failed - {}", pool, err),
 687                         }
 688                     }
 689                 }
 690                 _ => {
 691                     if let Ok(disk) = disk_manager.clone().disk_by_dev_num(device.into_dev_t()) {
 692                         match disk.read_stat() {
 693                             Ok(stat) => device_stat = stat,
 694                             Err(err) => eprintln!("disk.read_stat {:?} failed - {}", path, err),
 695                         }
 696                     }
 697                 }
 698             }
 699             if let Some(stat) = device_stat {
 700                 let rrd_key = format!("{}/read_ios", rrd_prefix);
 701                 rrd_update_derive(&rrd_key, stat.read_ios as f64, save);
 702                 let rrd_key = format!("{}/read_bytes", rrd_prefix);
 703                 rrd_update_derive(&rrd_key, (stat.read_sectors*512) as f64, save);
 704
 705                 let rrd_key = format!("{}/write_ios", rrd_prefix);
 706                 rrd_update_derive(&rrd_key, stat.write_ios as f64, save);
 707                 let rrd_key = format!("{}/write_bytes", rrd_prefix);
 708                 rrd_update_derive(&rrd_key, (stat.write_sectors*512) as f64, save);
 709
 710                 let rrd_key = format!("{}/io_ticks", rrd_prefix);
 711                 rrd_update_derive(&rrd_key, (stat.io_ticks as f64)/1000.0, save);
 712             }
 713         }
 714         Err(err) => {
 715             eprintln!("find_mounted_device failed - {}", err);
 716         }
 717     }
 718 }