2 use std
::path
::{Path, PathBuf}
;
4 use std
::os
::unix
::io
::AsRawFd
;
6 use anyhow
::{bail, format_err, Error}
;
9 use openssl
::ssl
::{SslMethod, SslAcceptor, SslFiletype}
;
10 use tokio_stream
::wrappers
::ReceiverStream
;
11 use serde_json
::Value
;
13 use proxmox
::try_block
;
14 use proxmox
::api
::RpcEnvironmentType
;
19 auth
::default_api_auth
,
27 rotate_task_log_archive
,
29 tools
::systemd
::time
::{
36 use proxmox_backup
::api2
::types
::Authid
;
37 use proxmox_backup
::configdir
;
38 use proxmox_backup
::buildcfg
;
39 use proxmox_backup
::server
;
40 use proxmox_backup
::auth_helpers
::*;
41 use proxmox_backup
::tools
::{
46 get_pool_from_dataset
,
51 PROXMOX_BACKUP_TCP_KEEPALIVE_TIME
,
55 use proxmox_backup
::api2
::pull
::do_sync_job
;
56 use proxmox_backup
::api2
::tape
::backup
::do_tape_backup_job
;
57 use proxmox_backup
::server
::do_verification_job
;
58 use proxmox_backup
::server
::do_prune_job
;
60 fn main() -> Result
<(), Error
> {
61 proxmox_backup
::tools
::setup_safe_path_env();
63 let backup_uid
= proxmox_backup
::backup
::backup_user()?
.uid
;
64 let backup_gid
= proxmox_backup
::backup
::backup_group()?
.gid
;
65 let running_uid
= nix
::unistd
::Uid
::effective();
66 let running_gid
= nix
::unistd
::Gid
::effective();
68 if running_uid
!= backup_uid
|| running_gid
!= backup_gid
{
69 bail
!("proxy not running as backup user or group (got uid {} gid {})", running_uid
, running_gid
);
72 proxmox_backup
::tools
::runtime
::main(run())
75 async
fn run() -> Result
<(), Error
> {
76 if let Err(err
) = syslog
::init(
77 syslog
::Facility
::LOG_DAEMON
,
78 log
::LevelFilter
::Info
,
79 Some("proxmox-backup-proxy")) {
80 bail
!("unable to inititialize syslog - {}", err
);
83 // Note: To debug early connection error use
84 // PROXMOX_DEBUG=1 ./target/release/proxmox-backup-proxy
85 let debug
= std
::env
::var("PROXMOX_DEBUG").is_ok();
87 let _
= public_auth_key(); // load with lazy_static
88 let _
= csrf_secret(); // load with lazy_static
90 let mut config
= ApiConfig
::new(
92 &proxmox_backup
::api2
::ROUTER
,
93 RpcEnvironmentType
::PUBLIC
,
97 config
.add_alias("novnc", "/usr/share/novnc-pve");
98 config
.add_alias("extjs", "/usr/share/javascript/extjs");
99 config
.add_alias("qrcodejs", "/usr/share/javascript/qrcodejs");
100 config
.add_alias("fontawesome", "/usr/share/fonts-font-awesome");
101 config
.add_alias("xtermjs", "/usr/share/pve-xtermjs");
102 config
.add_alias("locale", "/usr/share/pbs-i18n");
103 config
.add_alias("widgettoolkit", "/usr/share/javascript/proxmox-widget-toolkit");
104 config
.add_alias("docs", "/usr/share/doc/proxmox-backup/html");
106 let mut indexpath
= PathBuf
::from(buildcfg
::JS_DIR
);
107 indexpath
.push("index.hbs");
108 config
.register_template("index", &indexpath
)?
;
109 config
.register_template("console", "/usr/share/pve-xtermjs/index.html.hbs")?
;
111 let mut commando_sock
= server
::CommandoSocket
::new(server
::our_ctrl_sock());
113 config
.enable_file_log(buildcfg
::API_ACCESS_LOG_FN
, &mut commando_sock
)?
;
115 let rest_server
= RestServer
::new(config
);
117 //openssl req -x509 -newkey rsa:4096 -keyout /etc/proxmox-backup/proxy.key -out /etc/proxmox-backup/proxy.pem -nodes
119 // we build the initial acceptor here as we cannot start if this fails - certificate reloads
120 // will be handled inside the accept loop and simply log an error if we cannot load the new
122 let acceptor
= make_tls_acceptor()?
;
124 // to renew the acceptor we just let a command-socket handler trigger a Notify:
125 let notify_tls_cert_reload
= Arc
::new(tokio
::sync
::Notify
::new());
126 commando_sock
.register_command(
127 "reload-certificate".to_string(),
129 let notify_tls_cert_reload
= Arc
::clone(¬ify_tls_cert_reload
);
130 move |_value
| -> Result
<_
, Error
> {
131 notify_tls_cert_reload
.notify_one();
137 let server
= daemon
::create_daemon(
138 ([0,0,0,0,0,0,0,0], 8007).into(),
139 move |listener
, ready
| {
141 let connections
= accept_connections(listener
, acceptor
, debug
, notify_tls_cert_reload
);
142 let connections
= hyper
::server
::accept
::from_stream(ReceiverStream
::new(connections
));
145 .and_then(|_
| hyper
::Server
::builder(connections
)
147 .with_graceful_shutdown(server
::shutdown_future())
148 .map_err(Error
::from
)
150 .map_err(|err
| eprintln
!("server error: {}", err
))
154 "proxmox-backup-proxy.service",
157 server
::write_pid(buildcfg
::PROXMOX_BACKUP_PROXY_PID_FN
)?
;
158 daemon
::systemd_notify(daemon
::SystemdNotify
::Ready
)?
;
160 let init_result
: Result
<(), Error
> = try_block
!({
161 server
::register_task_control_commands(&mut commando_sock
)?
;
162 commando_sock
.spawn()?
;
163 server
::server_state_init()?
;
167 if let Err(err
) = init_result
{
168 bail
!("unable to start daemon - {}", err
);
171 start_task_scheduler();
172 start_stat_generator();
175 log
::info
!("server shutting down, waiting for active workers to complete");
176 proxmox_backup
::server
::last_worker_future().await?
;
177 log
::info
!("done - exit server");
182 fn make_tls_acceptor() -> Result
<Arc
<SslAcceptor
>, Error
> {
183 let key_path
= configdir
!("/proxy.key");
184 let cert_path
= configdir
!("/proxy.pem");
186 let mut acceptor
= SslAcceptor
::mozilla_intermediate_v5(SslMethod
::tls()).unwrap();
187 acceptor
.set_private_key_file(key_path
, SslFiletype
::PEM
)
188 .map_err(|err
| format_err
!("unable to read proxy key {} - {}", key_path
, err
))?
;
189 acceptor
.set_certificate_chain_file(cert_path
)
190 .map_err(|err
| format_err
!("unable to read proxy cert {} - {}", cert_path
, err
))?
;
191 acceptor
.check_private_key().unwrap();
193 Ok(Arc
::new(acceptor
.build()))
196 type ClientStreamResult
=
197 Result
<std
::pin
::Pin
<Box
<tokio_openssl
::SslStream
<tokio
::net
::TcpStream
>>>, Error
>;
198 const MAX_PENDING_ACCEPTS
: usize = 1024;
200 fn accept_connections(
201 listener
: tokio
::net
::TcpListener
,
202 acceptor
: Arc
<openssl
::ssl
::SslAcceptor
>,
204 notify_tls_cert_reload
: Arc
<tokio
::sync
::Notify
>,
205 ) -> tokio
::sync
::mpsc
::Receiver
<ClientStreamResult
> {
207 let (sender
, receiver
) = tokio
::sync
::mpsc
::channel(MAX_PENDING_ACCEPTS
);
209 tokio
::spawn(accept_connection(listener
, acceptor
, debug
, sender
, notify_tls_cert_reload
));
214 async
fn accept_connection(
215 listener
: tokio
::net
::TcpListener
,
216 mut acceptor
: Arc
<openssl
::ssl
::SslAcceptor
>,
218 sender
: tokio
::sync
::mpsc
::Sender
<ClientStreamResult
>,
219 notify_tls_cert_reload
: Arc
<tokio
::sync
::Notify
>,
221 let accept_counter
= Arc
::new(());
223 // Note that these must not be moved out/modified directly, they get pinned in the loop and
224 // "rearmed" after waking up:
225 let mut reload_tls
= notify_tls_cert_reload
.notified();
226 let mut accept
= listener
.accept();
231 // normally we'd use `tokio::pin!()` but we need this to happen outside the loop and we
232 // need to be able to "rearm" the futures:
233 let reload_tls_pin
= unsafe { Pin::new_unchecked(&mut reload_tls) }
;
234 let accept_pin
= unsafe { Pin::new_unchecked(&mut accept) }
;
236 _
= reload_tls_pin
=> {
237 // rearm the notification:
238 reload_tls
= notify_tls_cert_reload
.notified();
240 log
::info
!("reloading certificate");
241 match make_tls_acceptor() {
242 Err(err
) => eprintln
!("error reloading certificate: {}", err
),
243 Ok(new_acceptor
) => acceptor
= new_acceptor
,
247 res
= accept_pin
=> match res
{
249 eprintln
!("error accepting tcp connection: {}", err
);
252 Ok((new_sock
, _addr
)) => {
253 // rearm the accept future:
254 accept
= listener
.accept();
261 sock
.set_nodelay(true).unwrap();
262 let _
= set_tcp_keepalive(sock
.as_raw_fd(), PROXMOX_BACKUP_TCP_KEEPALIVE_TIME
);
263 let acceptor
= Arc
::clone(&acceptor
);
265 let ssl
= match openssl
::ssl
::Ssl
::new(acceptor
.context()) {
268 eprintln
!("failed to create Ssl object from Acceptor context - {}", err
);
272 let stream
= match tokio_openssl
::SslStream
::new(ssl
, sock
) {
273 Ok(stream
) => stream
,
275 eprintln
!("failed to create SslStream using ssl and connection socket - {}", err
);
280 let mut stream
= Box
::pin(stream
);
281 let sender
= sender
.clone();
283 if Arc
::strong_count(&accept_counter
) > MAX_PENDING_ACCEPTS
{
284 eprintln
!("connection rejected - to many open connections");
288 let accept_counter
= Arc
::clone(&accept_counter
);
289 tokio
::spawn(async
move {
290 let accept_future
= tokio
::time
::timeout(
291 Duration
::new(10, 0), stream
.as_mut().accept());
293 let result
= accept_future
.await
;
297 if sender
.send(Ok(stream
)).await
.is_err() && debug
{
298 eprintln
!("detect closed connection channel");
303 eprintln
!("https handshake failed - {}", err
);
308 eprintln
!("https handshake timeout");
313 drop(accept_counter
); // decrease reference count
318 fn start_stat_generator() {
319 let abort_future
= server
::shutdown_future();
320 let future
= Box
::pin(run_stat_generator());
321 let task
= futures
::future
::select(future
, abort_future
);
322 tokio
::spawn(task
.map(|_
| ()));
325 fn start_task_scheduler() {
326 let abort_future
= server
::shutdown_future();
327 let future
= Box
::pin(run_task_scheduler());
328 let task
= futures
::future
::select(future
, abort_future
);
329 tokio
::spawn(task
.map(|_
| ()));
332 use std
::time
::{SystemTime, Instant, Duration, UNIX_EPOCH}
;
334 fn next_minute() -> Result
<Instant
, Error
> {
335 let now
= SystemTime
::now();
336 let epoch_now
= now
.duration_since(UNIX_EPOCH
)?
;
337 let epoch_next
= Duration
::from_secs((epoch_now
.as_secs()/60 + 1)*60);
338 Ok(Instant
::now() + epoch_next
- epoch_now
)
341 async
fn run_task_scheduler() {
343 let mut count
: usize = 0;
348 let delay_target
= match next_minute() { // try to run very minute
351 eprintln
!("task scheduler: compute next minute failed - {}", err
);
352 tokio
::time
::sleep_until(tokio
::time
::Instant
::from_std(Instant
::now() + Duration
::from_secs(60))).await
;
357 if count
> 2 { // wait 1..2 minutes before starting
358 match schedule_tasks().catch_unwind().await
{
360 match panic
.downcast
::<&str>() {
362 eprintln
!("task scheduler panic: {}", msg
);
365 eprintln
!("task scheduler panic - unknown type");
370 eprintln
!("task scheduler failed - {:?}", err
);
376 tokio
::time
::sleep_until(tokio
::time
::Instant
::from_std(delay_target
)).await
;
380 async
fn schedule_tasks() -> Result
<(), Error
> {
382 schedule_datastore_garbage_collection().await
;
383 schedule_datastore_prune().await
;
384 schedule_datastore_sync_jobs().await
;
385 schedule_datastore_verify_jobs().await
;
386 schedule_tape_backup_jobs().await
;
387 schedule_task_log_rotate().await
;
392 async
fn schedule_datastore_garbage_collection() {
394 use proxmox_backup
::config
::{
401 let config
= match datastore
::config() {
403 eprintln
!("unable to read datastore config - {}", err
);
406 Ok((config
, _digest
)) => config
,
409 for (store
, (_
, store_config
)) in config
.sections
{
410 let datastore
= match DataStore
::lookup_datastore(&store
) {
411 Ok(datastore
) => datastore
,
413 eprintln
!("lookup_datastore failed - {}", err
);
418 let store_config
: DataStoreConfig
= match serde_json
::from_value(store_config
) {
421 eprintln
!("datastore config from_value failed - {}", err
);
426 let event_str
= match store_config
.gc_schedule
{
427 Some(event_str
) => event_str
,
431 let event
= match parse_calendar_event(&event_str
) {
434 eprintln
!("unable to parse schedule '{}' - {}", event_str
, err
);
439 if datastore
.garbage_collection_running() { continue; }
441 let worker_type
= "garbage_collection";
443 let last
= match jobstate
::last_run_time(worker_type
, &store
) {
446 eprintln
!("could not get last run time of {} {}: {}", worker_type
, store
, err
);
451 let next
= match compute_next_event(&event
, last
, false) {
452 Ok(Some(next
)) => next
,
453 Ok(None
) => continue,
455 eprintln
!("compute_next_event for '{}' failed - {}", event_str
, err
);
460 let now
= proxmox
::tools
::time
::epoch_i64();
462 if next
> now { continue; }
464 let job
= match Job
::new(worker_type
, &store
) {
466 Err(_
) => continue, // could not get lock
469 let auth_id
= Authid
::root_auth_id();
471 if let Err(err
) = crate::server
::do_garbage_collection_job(job
, datastore
, auth_id
, Some(event_str
), false) {
472 eprintln
!("unable to start garbage collection job on datastore {} - {}", store
, err
);
477 async
fn schedule_datastore_prune() {
479 use proxmox_backup
::{
489 let config
= match datastore
::config() {
491 eprintln
!("unable to read datastore config - {}", err
);
494 Ok((config
, _digest
)) => config
,
497 for (store
, (_
, store_config
)) in config
.sections
{
499 let store_config
: DataStoreConfig
= match serde_json
::from_value(store_config
) {
502 eprintln
!("datastore '{}' config from_value failed - {}", store
, err
);
507 let event_str
= match store_config
.prune_schedule
{
508 Some(event_str
) => event_str
,
512 let prune_options
= PruneOptions
{
513 keep_last
: store_config
.keep_last
,
514 keep_hourly
: store_config
.keep_hourly
,
515 keep_daily
: store_config
.keep_daily
,
516 keep_weekly
: store_config
.keep_weekly
,
517 keep_monthly
: store_config
.keep_monthly
,
518 keep_yearly
: store_config
.keep_yearly
,
521 if !prune_options
.keeps_something() { // no prune settings - keep all
525 let worker_type
= "prune";
526 if check_schedule(worker_type
, &event_str
, &store
) {
527 let job
= match Job
::new(worker_type
, &store
) {
529 Err(_
) => continue, // could not get lock
532 let auth_id
= Authid
::root_auth_id().clone();
533 if let Err(err
) = do_prune_job(job
, prune_options
, store
.clone(), &auth_id
, Some(event_str
)) {
534 eprintln
!("unable to start datastore prune job {} - {}", &store
, err
);
540 async
fn schedule_datastore_sync_jobs() {
542 use proxmox_backup
::config
::sync
::{
547 let config
= match sync
::config() {
549 eprintln
!("unable to read sync job config - {}", err
);
552 Ok((config
, _digest
)) => config
,
555 for (job_id
, (_
, job_config
)) in config
.sections
{
556 let job_config
: SyncJobConfig
= match serde_json
::from_value(job_config
) {
559 eprintln
!("sync job config from_value failed - {}", err
);
564 let event_str
= match job_config
.schedule
{
565 Some(ref event_str
) => event_str
.clone(),
569 let worker_type
= "syncjob";
570 if check_schedule(worker_type
, &event_str
, &job_id
) {
571 let job
= match Job
::new(worker_type
, &job_id
) {
573 Err(_
) => continue, // could not get lock
576 let auth_id
= Authid
::root_auth_id().clone();
577 if let Err(err
) = do_sync_job(job
, job_config
, &auth_id
, Some(event_str
)) {
578 eprintln
!("unable to start datastore sync job {} - {}", &job_id
, err
);
584 async
fn schedule_datastore_verify_jobs() {
586 use proxmox_backup
::config
::verify
::{
588 VerificationJobConfig
,
591 let config
= match verify
::config() {
593 eprintln
!("unable to read verification job config - {}", err
);
596 Ok((config
, _digest
)) => config
,
598 for (job_id
, (_
, job_config
)) in config
.sections
{
599 let job_config
: VerificationJobConfig
= match serde_json
::from_value(job_config
) {
602 eprintln
!("verification job config from_value failed - {}", err
);
606 let event_str
= match job_config
.schedule
{
607 Some(ref event_str
) => event_str
.clone(),
611 let worker_type
= "verificationjob";
612 let auth_id
= Authid
::root_auth_id().clone();
613 if check_schedule(worker_type
, &event_str
, &job_id
) {
614 let job
= match Job
::new(&worker_type
, &job_id
) {
616 Err(_
) => continue, // could not get lock
618 if let Err(err
) = do_verification_job(job
, job_config
, &auth_id
, Some(event_str
)) {
619 eprintln
!("unable to start datastore verification job {} - {}", &job_id
, err
);
625 async
fn schedule_tape_backup_jobs() {
627 use proxmox_backup
::config
::tape_job
::{
632 let config
= match tape_job
::config() {
634 eprintln
!("unable to read tape job config - {}", err
);
637 Ok((config
, _digest
)) => config
,
639 for (job_id
, (_
, job_config
)) in config
.sections
{
640 let job_config
: TapeBackupJobConfig
= match serde_json
::from_value(job_config
) {
643 eprintln
!("tape backup job config from_value failed - {}", err
);
647 let event_str
= match job_config
.schedule
{
648 Some(ref event_str
) => event_str
.clone(),
652 let worker_type
= "tape-backup-job";
653 let auth_id
= Authid
::root_auth_id().clone();
654 if check_schedule(worker_type
, &event_str
, &job_id
) {
655 let job
= match Job
::new(&worker_type
, &job_id
) {
657 Err(_
) => continue, // could not get lock
659 if let Err(err
) = do_tape_backup_job(job
, job_config
.setup
, &auth_id
, Some(event_str
)) {
660 eprintln
!("unable to start tape backup job {} - {}", &job_id
, err
);
667 async
fn schedule_task_log_rotate() {
669 let worker_type
= "logrotate";
670 let job_id
= "access-log_and_task-archive";
672 // schedule daily at 00:00 like normal logrotate
673 let schedule
= "00:00";
675 if !check_schedule(worker_type
, schedule
, job_id
) {
676 // if we never ran the rotation, schedule instantly
677 match jobstate
::JobState
::load(worker_type
, job_id
) {
678 Ok(state
) => match state
{
679 jobstate
::JobState
::Created { .. }
=> {}
,
686 let mut job
= match Job
::new(worker_type
, job_id
) {
688 Err(_
) => return, // could not get lock
691 if let Err(err
) = WorkerTask
::new_thread(
694 Authid
::root_auth_id().clone(),
697 job
.start(&worker
.upid().to_string())?
;
698 worker
.log("starting task log rotation".to_string());
700 let result
= try_block
!({
701 let max_size
= 512 * 1024 - 1; // an entry has ~ 100b, so > 5000 entries/file
702 let max_files
= 20; // times twenty files gives > 100000 task entries
703 let has_rotated
= rotate_task_log_archive(max_size
, true, Some(max_files
))?
;
705 worker
.log("task log archive was rotated".to_string());
707 worker
.log("task log archive was not rotated".to_string());
710 let max_size
= 32 * 1024 * 1024 - 1;
712 let mut logrotate
= LogRotate
::new(buildcfg
::API_ACCESS_LOG_FN
, true)
713 .ok_or_else(|| format_err
!("could not get API access log file names"))?
;
715 if logrotate
.rotate(max_size
, None
, Some(max_files
))?
{
716 println
!("rotated access log, telling daemons to re-open log file");
717 proxmox_backup
::tools
::runtime
::block_on(command_reopen_logfiles())?
;
718 worker
.log("API access log was rotated".to_string());
720 worker
.log("API access log was not rotated".to_string());
723 let mut logrotate
= LogRotate
::new(buildcfg
::API_AUTH_LOG_FN
, true)
724 .ok_or_else(|| format_err
!("could not get API auth log file names"))?
;
726 if logrotate
.rotate(max_size
, None
, Some(max_files
))?
{
727 worker
.log("API authentication log was rotated".to_string());
729 worker
.log("API authentication log was not rotated".to_string());
735 let status
= worker
.create_state(&result
);
737 if let Err(err
) = job
.finish(status
) {
738 eprintln
!("could not finish job state for {}: {}", worker_type
, err
);
744 eprintln
!("unable to start task log rotation: {}", err
);
749 async
fn command_reopen_logfiles() -> Result
<(), Error
> {
750 // only care about the most recent daemon instance for each, proxy & api, as other older ones
751 // should not respond to new requests anyway, but only finish their current one and then exit.
752 let sock
= server
::our_ctrl_sock();
753 let f1
= server
::send_command(sock
, "{\"command\":\"api-access-log-reopen\"}\n");
755 let pid
= server
::read_pid(buildcfg
::PROXMOX_BACKUP_API_PID_FN
)?
;
756 let sock
= server
::ctrl_sock_from_pid(pid
);
757 let f2
= server
::send_command(sock
, "{\"command\":\"api-access-log-reopen\"}\n");
759 match futures
::join
!(f1
, f2
) {
760 (Err(e1
), Err(e2
)) => Err(format_err
!("reopen commands failed, proxy: {}; api: {}", e1
, e2
)),
761 (Err(e1
), Ok(_
)) => Err(format_err
!("reopen commands failed, proxy: {}", e1
)),
762 (Ok(_
), Err(e2
)) => Err(format_err
!("reopen commands failed, api: {}", e2
)),
767 async
fn run_stat_generator() {
772 let save
= if count
>= 6 { count = 0; true }
else { false }
;
774 let delay_target
= Instant
::now() + Duration
::from_secs(10);
776 generate_host_stats(save
).await
;
778 tokio
::time
::sleep_until(tokio
::time
::Instant
::from_std(delay_target
)).await
;
784 fn rrd_update_gauge(name
: &str, value
: f64, save
: bool
) {
785 use proxmox_backup
::rrd
;
786 if let Err(err
) = rrd
::update_value(name
, value
, rrd
::DST
::Gauge
, save
) {
787 eprintln
!("rrd::update_value '{}' failed - {}", name
, err
);
791 fn rrd_update_derive(name
: &str, value
: f64, save
: bool
) {
792 use proxmox_backup
::rrd
;
793 if let Err(err
) = rrd
::update_value(name
, value
, rrd
::DST
::Derive
, save
) {
794 eprintln
!("rrd::update_value '{}' failed - {}", name
, err
);
798 async
fn generate_host_stats(save
: bool
) {
799 use proxmox
::sys
::linux
::procfs
::{
800 read_meminfo
, read_proc_stat
, read_proc_net_dev
, read_loadavg
};
801 use proxmox_backup
::config
::datastore
;
804 proxmox_backup
::tools
::runtime
::block_in_place(move || {
806 match read_proc_stat() {
808 rrd_update_gauge("host/cpu", stat
.cpu
, save
);
809 rrd_update_gauge("host/iowait", stat
.iowait_percent
, save
);
812 eprintln
!("read_proc_stat failed - {}", err
);
816 match read_meminfo() {
818 rrd_update_gauge("host/memtotal", meminfo
.memtotal
as f64, save
);
819 rrd_update_gauge("host/memused", meminfo
.memused
as f64, save
);
820 rrd_update_gauge("host/swaptotal", meminfo
.swaptotal
as f64, save
);
821 rrd_update_gauge("host/swapused", meminfo
.swapused
as f64, save
);
824 eprintln
!("read_meminfo failed - {}", err
);
828 match read_proc_net_dev() {
830 use proxmox_backup
::config
::network
::is_physical_nic
;
834 if !is_physical_nic(&item
.device
) { continue; }
835 netin
+= item
.receive
;
838 rrd_update_derive("host/netin", netin
as f64, save
);
839 rrd_update_derive("host/netout", netout
as f64, save
);
842 eprintln
!("read_prox_net_dev failed - {}", err
);
846 match read_loadavg() {
848 rrd_update_gauge("host/loadavg", loadavg
.0 as f64, save
);
851 eprintln
!("read_loadavg failed - {}", err
);
855 let disk_manager
= DiskManage
::new();
857 gather_disk_stats(disk_manager
.clone(), Path
::new("/"), "host", save
);
859 match datastore
::config() {
861 let datastore_list
: Vec
<datastore
::DataStoreConfig
> =
862 config
.convert_to_typed_array("datastore").unwrap_or_default();
864 for config
in datastore_list
{
866 let rrd_prefix
= format
!("datastore/{}", config
.name
);
867 let path
= std
::path
::Path
::new(&config
.path
);
868 gather_disk_stats(disk_manager
.clone(), path
, &rrd_prefix
, save
);
872 eprintln
!("read datastore config failed - {}", err
);
879 fn check_schedule(worker_type
: &str, event_str
: &str, id
: &str) -> bool
{
880 let event
= match parse_calendar_event(event_str
) {
883 eprintln
!("unable to parse schedule '{}' - {}", event_str
, err
);
888 let last
= match jobstate
::last_run_time(worker_type
, &id
) {
891 eprintln
!("could not get last run time of {} {}: {}", worker_type
, id
, err
);
896 let next
= match compute_next_event(&event
, last
, false) {
897 Ok(Some(next
)) => next
,
898 Ok(None
) => return false,
900 eprintln
!("compute_next_event for '{}' failed - {}", event_str
, err
);
905 let now
= proxmox
::tools
::time
::epoch_i64();
909 fn gather_disk_stats(disk_manager
: Arc
<DiskManage
>, path
: &Path
, rrd_prefix
: &str, save
: bool
) {
911 match proxmox_backup
::tools
::disks
::disk_usage(path
) {
913 let rrd_key
= format
!("{}/total", rrd_prefix
);
914 rrd_update_gauge(&rrd_key
, status
.total
as f64, save
);
915 let rrd_key
= format
!("{}/used", rrd_prefix
);
916 rrd_update_gauge(&rrd_key
, status
.used
as f64, save
);
919 eprintln
!("read disk_usage on {:?} failed - {}", path
, err
);
923 match disk_manager
.find_mounted_device(path
) {
925 Ok(Some((fs_type
, device
, source
))) => {
926 let mut device_stat
= None
;
927 match fs_type
.as_str() {
929 if let Some(source
) = source
{
930 let pool
= get_pool_from_dataset(&source
).unwrap_or(&source
);
931 match zfs_pool_stats(pool
) {
932 Ok(stat
) => device_stat
= stat
,
933 Err(err
) => eprintln
!("zfs_pool_stats({:?}) failed - {}", pool
, err
),
938 if let Ok(disk
) = disk_manager
.clone().disk_by_dev_num(device
.into_dev_t()) {
939 match disk
.read_stat() {
940 Ok(stat
) => device_stat
= stat
,
941 Err(err
) => eprintln
!("disk.read_stat {:?} failed - {}", path
, err
),
946 if let Some(stat
) = device_stat
{
947 let rrd_key
= format
!("{}/read_ios", rrd_prefix
);
948 rrd_update_derive(&rrd_key
, stat
.read_ios
as f64, save
);
949 let rrd_key
= format
!("{}/read_bytes", rrd_prefix
);
950 rrd_update_derive(&rrd_key
, (stat
.read_sectors
*512) as f64, save
);
952 let rrd_key
= format
!("{}/write_ios", rrd_prefix
);
953 rrd_update_derive(&rrd_key
, stat
.write_ios
as f64, save
);
954 let rrd_key
= format
!("{}/write_bytes", rrd_prefix
);
955 rrd_update_derive(&rrd_key
, (stat
.write_sectors
*512) as f64, save
);
957 let rrd_key
= format
!("{}/io_ticks", rrd_prefix
);
958 rrd_update_derive(&rrd_key
, (stat
.io_ticks
as f64)/1000.0, save
);
962 eprintln
!("find_mounted_device failed - {}", err
);