]> git.proxmox.com Git - proxmox-backup.git/blame - src/bin/proxmox-backup-proxy.rs
split out proxmox-backup-debug binary
[proxmox-backup.git] / src / bin / proxmox-backup-proxy.rs
CommitLineData
4ce7da51 1use std::sync::{Mutex, Arc};
2ab5acac 2use std::path::{Path, PathBuf};
97168f92 3use std::os::unix::io::AsRawFd;
a2479cfa 4
f7d4e4b5 5use anyhow::{bail, format_err, Error};
a2479cfa 6use futures::*;
ea368a06 7
a2479cfa 8use openssl::ssl::{SslMethod, SslAcceptor, SslFiletype};
7c667013 9use tokio_stream::wrappers::ReceiverStream;
a723c087 10use serde_json::Value;
a2479cfa 11
9ea4bce4 12use proxmox::try_block;
a2479cfa 13use proxmox::api::RpcEnvironmentType;
32413921 14use proxmox::sys::linux::socket::set_tcp_keepalive;
a2479cfa 15
1298618a
DM
16use proxmox_backup::{
17 backup::DataStore,
18 server::{
26858dba 19 auth::default_api_auth,
1298618a
DM
20 WorkerTask,
21 ApiConfig,
22 rest::*,
23 jobstate::{
24 self,
25 Job,
26 },
27 rotate_task_log_archive,
28 },
1298618a
DM
29};
30
af06decd 31use pbs_buildcfg::configdir;
dd2162f6 32use pbs_systemd::time::{compute_next_event, parse_calendar_event};
1298618a 33
1cd951c9 34use proxmox_backup::api2::types::Authid;
e3f41f21 35use proxmox_backup::server;
d01e2420 36use proxmox_backup::auth_helpers::*;
97168f92 37use proxmox_backup::tools::{
32413921 38 PROXMOX_BACKUP_TCP_KEEPALIVE_TIME,
e4f5f59e 39 daemon,
97168f92
DM
40 disks::{
41 DiskManage,
42 zfs_pool_stats,
368f4c54 43 get_pool_from_dataset,
97168f92 44 },
fe4cc5b1 45 logrotate::LogRotate,
97168f92 46};
02c7a755 47
a13573c2 48use proxmox_backup::api2::pull::do_sync_job;
8513626b 49use proxmox_backup::api2::tape::backup::do_tape_backup_job;
1298618a 50use proxmox_backup::server::do_verification_job;
b8d90798 51use proxmox_backup::server::do_prune_job;
a13573c2 52
946c3e8a 53fn main() -> Result<(), Error> {
ac7513e3
DM
54 proxmox_backup::tools::setup_safe_path_env();
55
843880f0
TL
56 let backup_uid = proxmox_backup::backup::backup_user()?.uid;
57 let backup_gid = proxmox_backup::backup::backup_group()?.gid;
58 let running_uid = nix::unistd::Uid::effective();
59 let running_gid = nix::unistd::Gid::effective();
60
61 if running_uid != backup_uid || running_gid != backup_gid {
62 bail!("proxy not running as backup user or group (got uid {} gid {})", running_uid, running_gid);
63 }
64
d420962f 65 pbs_runtime::main(run())
4223d9f8
DM
66}
67
fda5797b 68async fn run() -> Result<(), Error> {
02c7a755
DM
69 if let Err(err) = syslog::init(
70 syslog::Facility::LOG_DAEMON,
71 log::LevelFilter::Info,
72 Some("proxmox-backup-proxy")) {
4223d9f8 73 bail!("unable to inititialize syslog - {}", err);
02c7a755
DM
74 }
75
e1d367df
DM
76 // Note: To debug early connection error use
77 // PROXMOX_DEBUG=1 ./target/release/proxmox-backup-proxy
78 let debug = std::env::var("PROXMOX_DEBUG").is_ok();
79
d01e2420
DM
80 let _ = public_auth_key(); // load with lazy_static
81 let _ = csrf_secret(); // load with lazy_static
82
02c7a755 83 let mut config = ApiConfig::new(
af06decd 84 pbs_buildcfg::JS_DIR,
26858dba
SR
85 &proxmox_backup::api2::ROUTER,
86 RpcEnvironmentType::PUBLIC,
87 default_api_auth(),
88 )?;
02c7a755 89
02c7a755
DM
90 config.add_alias("novnc", "/usr/share/novnc-pve");
91 config.add_alias("extjs", "/usr/share/javascript/extjs");
7f066a9b 92 config.add_alias("qrcodejs", "/usr/share/javascript/qrcodejs");
02c7a755
DM
93 config.add_alias("fontawesome", "/usr/share/fonts-font-awesome");
94 config.add_alias("xtermjs", "/usr/share/pve-xtermjs");
abd4c4cb 95 config.add_alias("locale", "/usr/share/pbs-i18n");
02c7a755 96 config.add_alias("widgettoolkit", "/usr/share/javascript/proxmox-widget-toolkit");
9c01e73c 97 config.add_alias("docs", "/usr/share/doc/proxmox-backup/html");
02c7a755 98
af06decd 99 let mut indexpath = PathBuf::from(pbs_buildcfg::JS_DIR);
2ab5acac
DC
100 indexpath.push("index.hbs");
101 config.register_template("index", &indexpath)?;
01ca99da 102 config.register_template("console", "/usr/share/pve-xtermjs/index.html.hbs")?;
2ab5acac 103
a68768cf
TL
104 let mut commando_sock = server::CommandoSocket::new(server::our_ctrl_sock());
105
af06decd 106 config.enable_file_log(pbs_buildcfg::API_ACCESS_LOG_FN, &mut commando_sock)?;
8e7e2223 107
02c7a755
DM
108 let rest_server = RestServer::new(config);
109
6d1f61b2 110 //openssl req -x509 -newkey rsa:4096 -keyout /etc/proxmox-backup/proxy.key -out /etc/proxmox-backup/proxy.pem -nodes
6d1f61b2 111
4ce7da51 112 // we build the initial acceptor here as we cannot start if this fails
c381a162 113 let acceptor = make_tls_acceptor()?;
4ce7da51 114 let acceptor = Arc::new(Mutex::new(acceptor));
6d1f61b2 115
4ce7da51 116 // to renew the acceptor we just add a command-socket handler
a723c087
WB
117 commando_sock.register_command(
118 "reload-certificate".to_string(),
119 {
4ce7da51 120 let acceptor = Arc::clone(&acceptor);
a723c087 121 move |_value| -> Result<_, Error> {
4ce7da51
DM
122 log::info!("reloading certificate");
123 match make_tls_acceptor() {
124 Err(err) => log::error!("error reloading certificate: {}", err),
125 Ok(new_acceptor) => {
126 let mut guard = acceptor.lock().unwrap();
127 *guard = new_acceptor;
128 }
129 }
a723c087
WB
130 Ok(Value::Null)
131 }
132 },
133 )?;
0d176f36 134
062cf75c
DC
135 // to remove references for not configured datastores
136 commando_sock.register_command(
137 "datastore-removed".to_string(),
138 |_value| {
139 if let Err(err) = proxmox_backup::backup::DataStore::remove_unused_datastores() {
140 log::error!("could not refresh datastores: {}", err);
141 }
142 Ok(Value::Null)
143 }
144 )?;
145
a690ecac
WB
146 let server = daemon::create_daemon(
147 ([0,0,0,0,0,0,0,0], 8007).into(),
a723c087 148 move |listener, ready| {
97168f92 149
4ce7da51 150 let connections = accept_connections(listener, acceptor, debug);
7c667013 151 let connections = hyper::server::accept::from_stream(ReceiverStream::new(connections));
083ff3fd
WB
152
153 Ok(ready
48aa2b93 154 .and_then(|_| hyper::Server::builder(connections)
083ff3fd
WB
155 .serve(rest_server)
156 .with_graceful_shutdown(server::shutdown_future())
157 .map_err(Error::from)
158 )
159 .map_err(|err| eprintln!("server error: {}", err))
160 .map(|_| ())
a690ecac 161 )
a2ca7137 162 },
d7c6ad60 163 "proxmox-backup-proxy.service",
083ff3fd 164 );
a2ca7137 165
af06decd 166 server::write_pid(pbs_buildcfg::PROXMOX_BACKUP_PROXY_PID_FN)?;
d98c9a7a
WB
167 daemon::systemd_notify(daemon::SystemdNotify::Ready)?;
168
fda5797b 169 let init_result: Result<(), Error> = try_block!({
a68768cf
TL
170 server::register_task_control_commands(&mut commando_sock)?;
171 commando_sock.spawn()?;
fda5797b
WB
172 server::server_state_init()?;
173 Ok(())
174 });
d607b886 175
fda5797b
WB
176 if let Err(err) = init_result {
177 bail!("unable to start daemon - {}", err);
178 }
e3f41f21 179
8545480a 180 start_task_scheduler();
eaeda365 181 start_stat_generator();
8545480a 182
083ff3fd 183 server.await?;
a546a8a0
WB
184 log::info!("server shutting down, waiting for active workers to complete");
185 proxmox_backup::server::last_worker_future().await?;
fda5797b 186 log::info!("done - exit server");
e3f41f21 187
4223d9f8 188 Ok(())
02c7a755 189}
8545480a 190
4ce7da51 191fn make_tls_acceptor() -> Result<SslAcceptor, Error> {
c381a162
WB
192 let key_path = configdir!("/proxy.key");
193 let cert_path = configdir!("/proxy.pem");
194
195 let mut acceptor = SslAcceptor::mozilla_intermediate_v5(SslMethod::tls()).unwrap();
196 acceptor.set_private_key_file(key_path, SslFiletype::PEM)
197 .map_err(|err| format_err!("unable to read proxy key {} - {}", key_path, err))?;
198 acceptor.set_certificate_chain_file(cert_path)
199 .map_err(|err| format_err!("unable to read proxy cert {} - {}", cert_path, err))?;
200 acceptor.check_private_key().unwrap();
201
4ce7da51 202 Ok(acceptor.build())
c381a162
WB
203}
204
a5e3be49
WB
205type ClientStreamResult =
206 Result<std::pin::Pin<Box<tokio_openssl::SslStream<tokio::net::TcpStream>>>, Error>;
207const MAX_PENDING_ACCEPTS: usize = 1024;
208
48aa2b93 209fn accept_connections(
0bfcea6a 210 listener: tokio::net::TcpListener,
4ce7da51 211 acceptor: Arc<Mutex<openssl::ssl::SslAcceptor>>,
e1d367df 212 debug: bool,
a5e3be49 213) -> tokio::sync::mpsc::Receiver<ClientStreamResult> {
48aa2b93 214
ea93bea7 215 let (sender, receiver) = tokio::sync::mpsc::channel(MAX_PENDING_ACCEPTS);
48aa2b93 216
4ce7da51 217 tokio::spawn(accept_connection(listener, acceptor, debug, sender));
a5e3be49
WB
218
219 receiver
220}
221
222async fn accept_connection(
223 listener: tokio::net::TcpListener,
4ce7da51 224 acceptor: Arc<Mutex<openssl::ssl::SslAcceptor>>,
a5e3be49
WB
225 debug: bool,
226 sender: tokio::sync::mpsc::Sender<ClientStreamResult>,
227) {
ea93bea7 228 let accept_counter = Arc::new(());
48aa2b93 229
a5e3be49 230 loop {
4ce7da51
DM
231 let (sock, _addr) = match listener.accept().await {
232 Ok(conn) => conn,
233 Err(err) => {
234 eprintln!("error accepting tcp connection: {}", err);
cc269b9f 235 continue;
a5e3be49 236 }
cc269b9f 237 };
48aa2b93 238
cc269b9f
WB
239 sock.set_nodelay(true).unwrap();
240 let _ = set_tcp_keepalive(sock.as_raw_fd(), PROXMOX_BACKUP_TCP_KEEPALIVE_TIME);
48aa2b93 241
4ce7da51
DM
242 let ssl = { // limit acceptor_guard scope
243 // Acceptor can be reloaded using the command socket "reload-certificate" command
244 let acceptor_guard = acceptor.lock().unwrap();
245
246 match openssl::ssl::Ssl::new(acceptor_guard.context()) {
247 Ok(ssl) => ssl,
248 Err(err) => {
249 eprintln!("failed to create Ssl object from Acceptor context - {}", err);
250 continue;
251 },
252 }
cc269b9f 253 };
4ce7da51 254
cc269b9f
WB
255 let stream = match tokio_openssl::SslStream::new(ssl, sock) {
256 Ok(stream) => stream,
257 Err(err) => {
258 eprintln!("failed to create SslStream using ssl and connection socket - {}", err);
259 continue;
260 },
261 };
262
263 let mut stream = Box::pin(stream);
264 let sender = sender.clone();
265
266 if Arc::strong_count(&accept_counter) > MAX_PENDING_ACCEPTS {
267 eprintln!("connection rejected - to many open connections");
268 continue;
48aa2b93 269 }
cc269b9f 270
b4931192 271 let accept_counter = Arc::clone(&accept_counter);
cc269b9f
WB
272 tokio::spawn(async move {
273 let accept_future = tokio::time::timeout(
274 Duration::new(10, 0), stream.as_mut().accept());
275
276 let result = accept_future.await;
277
278 match result {
279 Ok(Ok(())) => {
280 if sender.send(Ok(stream)).await.is_err() && debug {
281 eprintln!("detect closed connection channel");
282 }
283 }
284 Ok(Err(err)) => {
285 if debug {
286 eprintln!("https handshake failed - {}", err);
287 }
288 }
289 Err(_) => {
290 if debug {
291 eprintln!("https handshake timeout");
292 }
293 }
294 }
295
296 drop(accept_counter); // decrease reference count
297 });
a5e3be49 298 }
48aa2b93
DM
299}
300
eaeda365
DM
301fn start_stat_generator() {
302 let abort_future = server::shutdown_future();
303 let future = Box::pin(run_stat_generator());
304 let task = futures::future::select(future, abort_future);
305 tokio::spawn(task.map(|_| ()));
306}
307
8545480a
DM
308fn start_task_scheduler() {
309 let abort_future = server::shutdown_future();
310 let future = Box::pin(run_task_scheduler());
311 let task = futures::future::select(future, abort_future);
312 tokio::spawn(task.map(|_| ()));
313}
314
6a7be83e 315use std::time::{SystemTime, Instant, Duration, UNIX_EPOCH};
8545480a
DM
316
317fn next_minute() -> Result<Instant, Error> {
6a7be83e
DM
318 let now = SystemTime::now();
319 let epoch_now = now.duration_since(UNIX_EPOCH)?;
320 let epoch_next = Duration::from_secs((epoch_now.as_secs()/60 + 1)*60);
8545480a
DM
321 Ok(Instant::now() + epoch_next - epoch_now)
322}
323
324async fn run_task_scheduler() {
325
326 let mut count: usize = 0;
327
328 loop {
329 count += 1;
330
331 let delay_target = match next_minute() { // try to run very minute
332 Ok(d) => d,
333 Err(err) => {
334 eprintln!("task scheduler: compute next minute failed - {}", err);
0a8d773a 335 tokio::time::sleep_until(tokio::time::Instant::from_std(Instant::now() + Duration::from_secs(60))).await;
8545480a
DM
336 continue;
337 }
338 };
339
340 if count > 2 { // wait 1..2 minutes before starting
341 match schedule_tasks().catch_unwind().await {
342 Err(panic) => {
343 match panic.downcast::<&str>() {
344 Ok(msg) => {
345 eprintln!("task scheduler panic: {}", msg);
346 }
347 Err(_) => {
348 eprintln!("task scheduler panic - unknown type");
349 }
350 }
351 }
352 Ok(Err(err)) => {
353 eprintln!("task scheduler failed - {:?}", err);
354 }
355 Ok(Ok(_)) => {}
356 }
357 }
358
0a8d773a 359 tokio::time::sleep_until(tokio::time::Instant::from_std(delay_target)).await;
8545480a
DM
360 }
361}
362
363async fn schedule_tasks() -> Result<(), Error> {
364
365 schedule_datastore_garbage_collection().await;
25829a87 366 schedule_datastore_prune().await;
a6160cdf 367 schedule_datastore_sync_jobs().await;
73df9c51 368 schedule_datastore_verify_jobs().await;
8513626b 369 schedule_tape_backup_jobs().await;
9a760917 370 schedule_task_log_rotate().await;
8545480a
DM
371
372 Ok(())
373}
374
8545480a
DM
375async fn schedule_datastore_garbage_collection() {
376
b9e7bcc2
DM
377 use proxmox_backup::config::{
378 datastore::{
379 self,
380 DataStoreConfig,
381 },
d7a122a0 382 };
8545480a 383
25829a87 384 let config = match datastore::config() {
8545480a
DM
385 Err(err) => {
386 eprintln!("unable to read datastore config - {}", err);
387 return;
388 }
389 Ok((config, _digest)) => config,
390 };
391
392 for (store, (_, store_config)) in config.sections {
393 let datastore = match DataStore::lookup_datastore(&store) {
394 Ok(datastore) => datastore,
395 Err(err) => {
396 eprintln!("lookup_datastore failed - {}", err);
397 continue;
398 }
399 };
400
25829a87 401 let store_config: DataStoreConfig = match serde_json::from_value(store_config) {
8545480a
DM
402 Ok(c) => c,
403 Err(err) => {
404 eprintln!("datastore config from_value failed - {}", err);
405 continue;
406 }
407 };
408
409 let event_str = match store_config.gc_schedule {
410 Some(event_str) => event_str,
411 None => continue,
412 };
413
414 let event = match parse_calendar_event(&event_str) {
415 Ok(event) => event,
416 Err(err) => {
417 eprintln!("unable to parse schedule '{}' - {}", event_str, err);
418 continue;
419 }
420 };
421
422 if datastore.garbage_collection_running() { continue; }
423
424 let worker_type = "garbage_collection";
425
b6ba5acd
DC
426 let last = match jobstate::last_run_time(worker_type, &store) {
427 Ok(time) => time,
428 Err(err) => {
429 eprintln!("could not get last run time of {} {}: {}", worker_type, store, err);
430 continue;
8545480a
DM
431 }
432 };
433
434 let next = match compute_next_event(&event, last, false) {
15ec790a
DC
435 Ok(Some(next)) => next,
436 Ok(None) => continue,
8545480a
DM
437 Err(err) => {
438 eprintln!("compute_next_event for '{}' failed - {}", event_str, err);
439 continue;
440 }
441 };
e693818a 442
6a7be83e
DM
443 let now = proxmox::tools::time::epoch_i64();
444
8545480a
DM
445 if next > now { continue; }
446
1cd951c9 447 let job = match Job::new(worker_type, &store) {
d7a122a0
DC
448 Ok(job) => job,
449 Err(_) => continue, // could not get lock
450 };
451
ad54df31 452 let auth_id = Authid::root_auth_id();
d7a122a0 453
c724f658 454 if let Err(err) = crate::server::do_garbage_collection_job(job, datastore, auth_id, Some(event_str), false) {
3b707fbb 455 eprintln!("unable to start garbage collection job on datastore {} - {}", store, err);
8545480a
DM
456 }
457 }
458}
25829a87
DM
459
460async fn schedule_datastore_prune() {
461
b2065dc7 462 use pbs_datastore::prune::PruneOptions;
1298618a 463 use proxmox_backup::{
1298618a
DM
464 config::datastore::{
465 self,
466 DataStoreConfig,
467 },
9866de5e 468 };
25829a87
DM
469
470 let config = match datastore::config() {
471 Err(err) => {
472 eprintln!("unable to read datastore config - {}", err);
473 return;
474 }
475 Ok((config, _digest)) => config,
476 };
477
478 for (store, (_, store_config)) in config.sections {
25829a87
DM
479
480 let store_config: DataStoreConfig = match serde_json::from_value(store_config) {
481 Ok(c) => c,
482 Err(err) => {
a6160cdf 483 eprintln!("datastore '{}' config from_value failed - {}", store, err);
25829a87
DM
484 continue;
485 }
486 };
487
488 let event_str = match store_config.prune_schedule {
489 Some(event_str) => event_str,
490 None => continue,
491 };
492
493 let prune_options = PruneOptions {
494 keep_last: store_config.keep_last,
495 keep_hourly: store_config.keep_hourly,
496 keep_daily: store_config.keep_daily,
497 keep_weekly: store_config.keep_weekly,
498 keep_monthly: store_config.keep_monthly,
499 keep_yearly: store_config.keep_yearly,
500 };
501
502 if !prune_options.keeps_something() { // no prune settings - keep all
503 continue;
504 }
505
25829a87 506 let worker_type = "prune";
b15751bf 507 if check_schedule(worker_type, &event_str, &store) {
82c05b41
HL
508 let job = match Job::new(worker_type, &store) {
509 Ok(job) => job,
510 Err(_) => continue, // could not get lock
511 };
25829a87 512
ad54df31 513 let auth_id = Authid::root_auth_id().clone();
82c05b41
HL
514 if let Err(err) = do_prune_job(job, prune_options, store.clone(), &auth_id, Some(event_str)) {
515 eprintln!("unable to start datastore prune job {} - {}", &store, err);
25829a87
DM
516 }
517 };
25829a87
DM
518 }
519}
a6160cdf
DM
520
521async fn schedule_datastore_sync_jobs() {
522
1298618a
DM
523 use proxmox_backup::config::sync::{
524 self,
525 SyncJobConfig,
a6160cdf
DM
526 };
527
528 let config = match sync::config() {
529 Err(err) => {
530 eprintln!("unable to read sync job config - {}", err);
531 return;
532 }
533 Ok((config, _digest)) => config,
534 };
535
a6160cdf
DM
536 for (job_id, (_, job_config)) in config.sections {
537 let job_config: SyncJobConfig = match serde_json::from_value(job_config) {
538 Ok(c) => c,
539 Err(err) => {
540 eprintln!("sync job config from_value failed - {}", err);
541 continue;
542 }
543 };
544
545 let event_str = match job_config.schedule {
546 Some(ref event_str) => event_str.clone(),
547 None => continue,
548 };
549
c67b1fa7 550 let worker_type = "syncjob";
b15751bf 551 if check_schedule(worker_type, &event_str, &job_id) {
82c05b41
HL
552 let job = match Job::new(worker_type, &job_id) {
553 Ok(job) => job,
554 Err(_) => continue, // could not get lock
555 };
a6160cdf 556
ad54df31 557 let auth_id = Authid::root_auth_id().clone();
82c05b41
HL
558 if let Err(err) = do_sync_job(job, job_config, &auth_id, Some(event_str)) {
559 eprintln!("unable to start datastore sync job {} - {}", &job_id, err);
a6160cdf
DM
560 }
561 };
a6160cdf
DM
562 }
563}
eaeda365 564
73df9c51 565async fn schedule_datastore_verify_jobs() {
1298618a
DM
566
567 use proxmox_backup::config::verify::{
568 self,
569 VerificationJobConfig,
73df9c51 570 };
1298618a 571
73df9c51
HL
572 let config = match verify::config() {
573 Err(err) => {
574 eprintln!("unable to read verification job config - {}", err);
575 return;
576 }
577 Ok((config, _digest)) => config,
578 };
579 for (job_id, (_, job_config)) in config.sections {
580 let job_config: VerificationJobConfig = match serde_json::from_value(job_config) {
581 Ok(c) => c,
582 Err(err) => {
583 eprintln!("verification job config from_value failed - {}", err);
584 continue;
585 }
586 };
587 let event_str = match job_config.schedule {
588 Some(ref event_str) => event_str.clone(),
589 None => continue,
590 };
82c05b41 591
73df9c51 592 let worker_type = "verificationjob";
ad54df31 593 let auth_id = Authid::root_auth_id().clone();
b15751bf 594 if check_schedule(worker_type, &event_str, &job_id) {
82c05b41
HL
595 let job = match Job::new(&worker_type, &job_id) {
596 Ok(job) => job,
597 Err(_) => continue, // could not get lock
598 };
599 if let Err(err) = do_verification_job(job, job_config, &auth_id, Some(event_str)) {
600 eprintln!("unable to start datastore verification job {} - {}", &job_id, err);
73df9c51
HL
601 }
602 };
73df9c51
HL
603 }
604}
605
8513626b
DM
606async fn schedule_tape_backup_jobs() {
607
608 use proxmox_backup::config::tape_job::{
609 self,
610 TapeBackupJobConfig,
611 };
612
613 let config = match tape_job::config() {
614 Err(err) => {
615 eprintln!("unable to read tape job config - {}", err);
616 return;
617 }
618 Ok((config, _digest)) => config,
619 };
620 for (job_id, (_, job_config)) in config.sections {
621 let job_config: TapeBackupJobConfig = match serde_json::from_value(job_config) {
622 Ok(c) => c,
623 Err(err) => {
624 eprintln!("tape backup job config from_value failed - {}", err);
625 continue;
626 }
627 };
628 let event_str = match job_config.schedule {
629 Some(ref event_str) => event_str.clone(),
630 None => continue,
631 };
632
633 let worker_type = "tape-backup-job";
634 let auth_id = Authid::root_auth_id().clone();
635 if check_schedule(worker_type, &event_str, &job_id) {
636 let job = match Job::new(&worker_type, &job_id) {
637 Ok(job) => job,
638 Err(_) => continue, // could not get lock
639 };
5830e562 640 if let Err(err) = do_tape_backup_job(job, job_config.setup, &auth_id, Some(event_str)) {
7a61f89e 641 eprintln!("unable to start tape backup job {} - {}", &job_id, err);
8513626b
DM
642 }
643 };
644 }
645}
646
647
9a760917 648async fn schedule_task_log_rotate() {
9a760917
DC
649
650 let worker_type = "logrotate";
72aa1834 651 let job_id = "access-log_and_task-archive";
9a760917 652
9a760917
DC
653 // schedule daily at 00:00 like normal logrotate
654 let schedule = "00:00";
655
b15751bf 656 if !check_schedule(worker_type, schedule, job_id) {
9a760917
DC
657 // if we never ran the rotation, schedule instantly
658 match jobstate::JobState::load(worker_type, job_id) {
659 Ok(state) => match state {
660 jobstate::JobState::Created { .. } => {},
661 _ => return,
662 },
663 _ => return,
664 }
665 }
666
667 let mut job = match Job::new(worker_type, job_id) {
668 Ok(job) => job,
669 Err(_) => return, // could not get lock
670 };
671
672 if let Err(err) = WorkerTask::new_thread(
673 worker_type,
72aa1834 674 None,
ad54df31 675 Authid::root_auth_id().clone(),
9a760917
DC
676 false,
677 move |worker| {
678 job.start(&worker.upid().to_string())?;
3b82f3ee 679 worker.log("starting task log rotation".to_string());
e4f5f59e 680
9a760917 681 let result = try_block!({
b7f2be51
TL
682 let max_size = 512 * 1024 - 1; // an entry has ~ 100b, so > 5000 entries/file
683 let max_files = 20; // times twenty files gives > 100000 task entries
9a760917
DC
684 let has_rotated = rotate_task_log_archive(max_size, true, Some(max_files))?;
685 if has_rotated {
3b82f3ee 686 worker.log("task log archive was rotated".to_string());
9a760917 687 } else {
3b82f3ee 688 worker.log("task log archive was not rotated".to_string());
9a760917
DC
689 }
690
fe4cc5b1
TL
691 let max_size = 32 * 1024 * 1024 - 1;
692 let max_files = 14;
af06decd 693 let mut logrotate = LogRotate::new(pbs_buildcfg::API_ACCESS_LOG_FN, true)
fe4cc5b1
TL
694 .ok_or_else(|| format_err!("could not get API access log file names"))?;
695
fe7bdc9d 696 if logrotate.rotate(max_size, None, Some(max_files))? {
fe4cc5b1 697 println!("rotated access log, telling daemons to re-open log file");
d420962f 698 pbs_runtime::block_on(command_reopen_logfiles())?;
3b82f3ee 699 worker.log("API access log was rotated".to_string());
fe7bdc9d 700 } else {
3b82f3ee 701 worker.log("API access log was not rotated".to_string());
fe7bdc9d
TL
702 }
703
af06decd 704 let mut logrotate = LogRotate::new(pbs_buildcfg::API_AUTH_LOG_FN, true)
fe7bdc9d 705 .ok_or_else(|| format_err!("could not get API auth log file names"))?;
fe4cc5b1 706
fe7bdc9d 707 if logrotate.rotate(max_size, None, Some(max_files))? {
3b82f3ee 708 worker.log("API authentication log was rotated".to_string());
fe4cc5b1 709 } else {
3b82f3ee 710 worker.log("API authentication log was not rotated".to_string());
fe4cc5b1
TL
711 }
712
9a760917
DC
713 Ok(())
714 });
715
716 let status = worker.create_state(&result);
717
718 if let Err(err) = job.finish(status) {
719 eprintln!("could not finish job state for {}: {}", worker_type, err);
720 }
721
722 result
723 },
724 ) {
725 eprintln!("unable to start task log rotation: {}", err);
726 }
727
728}
729
fe4cc5b1
TL
730async fn command_reopen_logfiles() -> Result<(), Error> {
731 // only care about the most recent daemon instance for each, proxy & api, as other older ones
732 // should not respond to new requests anyway, but only finish their current one and then exit.
733 let sock = server::our_ctrl_sock();
45b8a032 734 let f1 = server::send_command(sock, "{\"command\":\"api-access-log-reopen\"}\n");
fe4cc5b1 735
af06decd 736 let pid = server::read_pid(pbs_buildcfg::PROXMOX_BACKUP_API_PID_FN)?;
fe4cc5b1 737 let sock = server::ctrl_sock_from_pid(pid);
45b8a032 738 let f2 = server::send_command(sock, "{\"command\":\"api-access-log-reopen\"}\n");
546b6a23
TL
739
740 match futures::join!(f1, f2) {
741 (Err(e1), Err(e2)) => Err(format_err!("reopen commands failed, proxy: {}; api: {}", e1, e2)),
742 (Err(e1), Ok(_)) => Err(format_err!("reopen commands failed, proxy: {}", e1)),
743 (Ok(_), Err(e2)) => Err(format_err!("reopen commands failed, api: {}", e2)),
744 _ => Ok(()),
745 }
fe4cc5b1
TL
746}
747
eaeda365
DM
748async fn run_stat_generator() {
749
013fa7bb 750 let mut count = 0;
eaeda365 751 loop {
013fa7bb 752 count += 1;
a720894f 753 let save = if count >= 6 { count = 0; true } else { false };
013fa7bb 754
eaeda365
DM
755 let delay_target = Instant::now() + Duration::from_secs(10);
756
013fa7bb 757 generate_host_stats(save).await;
eaeda365 758
0a8d773a 759 tokio::time::sleep_until(tokio::time::Instant::from_std(delay_target)).await;
013fa7bb
DM
760
761 }
eaeda365
DM
762
763}
764
013fa7bb 765fn rrd_update_gauge(name: &str, value: f64, save: bool) {
309ef20d 766 use proxmox_backup::rrd;
013fa7bb 767 if let Err(err) = rrd::update_value(name, value, rrd::DST::Gauge, save) {
309ef20d
DM
768 eprintln!("rrd::update_value '{}' failed - {}", name, err);
769 }
770}
771
013fa7bb 772fn rrd_update_derive(name: &str, value: f64, save: bool) {
309ef20d 773 use proxmox_backup::rrd;
013fa7bb 774 if let Err(err) = rrd::update_value(name, value, rrd::DST::Derive, save) {
309ef20d
DM
775 eprintln!("rrd::update_value '{}' failed - {}", name, err);
776 }
777}
778
013fa7bb 779async fn generate_host_stats(save: bool) {
8f0cec26 780 use proxmox::sys::linux::procfs::{
485841da 781 read_meminfo, read_proc_stat, read_proc_net_dev, read_loadavg};
309ef20d 782 use proxmox_backup::config::datastore;
8c03041a 783
eaeda365 784
d420962f 785 pbs_runtime::block_in_place(move || {
4f951399
DM
786
787 match read_proc_stat() {
788 Ok(stat) => {
013fa7bb
DM
789 rrd_update_gauge("host/cpu", stat.cpu, save);
790 rrd_update_gauge("host/iowait", stat.iowait_percent, save);
4f951399
DM
791 }
792 Err(err) => {
793 eprintln!("read_proc_stat failed - {}", err);
eaeda365
DM
794 }
795 }
2c66a590 796
4f951399
DM
797 match read_meminfo() {
798 Ok(meminfo) => {
013fa7bb
DM
799 rrd_update_gauge("host/memtotal", meminfo.memtotal as f64, save);
800 rrd_update_gauge("host/memused", meminfo.memused as f64, save);
801 rrd_update_gauge("host/swaptotal", meminfo.swaptotal as f64, save);
802 rrd_update_gauge("host/swapused", meminfo.swapused as f64, save);
a4a3f7ca 803 }
4f951399
DM
804 Err(err) => {
805 eprintln!("read_meminfo failed - {}", err);
a4a3f7ca
DM
806 }
807 }
8f0cec26 808
4f951399
DM
809 match read_proc_net_dev() {
810 Ok(netdev) => {
811 use proxmox_backup::config::network::is_physical_nic;
812 let mut netin = 0;
813 let mut netout = 0;
814 for item in netdev {
815 if !is_physical_nic(&item.device) { continue; }
816 netin += item.receive;
817 netout += item.send;
818 }
013fa7bb
DM
819 rrd_update_derive("host/netin", netin as f64, save);
820 rrd_update_derive("host/netout", netout as f64, save);
8f0cec26 821 }
4f951399
DM
822 Err(err) => {
823 eprintln!("read_prox_net_dev failed - {}", err);
8f0cec26
DM
824 }
825 }
dd15c0aa 826
485841da
DM
827 match read_loadavg() {
828 Ok(loadavg) => {
013fa7bb 829 rrd_update_gauge("host/loadavg", loadavg.0 as f64, save);
485841da
DM
830 }
831 Err(err) => {
832 eprintln!("read_loadavg failed - {}", err);
833 }
834 }
835
8c03041a
DM
836 let disk_manager = DiskManage::new();
837
013fa7bb 838 gather_disk_stats(disk_manager.clone(), Path::new("/"), "host", save);
91e5bb49 839
d0833a70
DM
840 match datastore::config() {
841 Ok((config, _)) => {
842 let datastore_list: Vec<datastore::DataStoreConfig> =
17c7b46a 843 config.convert_to_typed_array("datastore").unwrap_or_default();
d0833a70
DM
844
845 for config in datastore_list {
8c03041a 846
91e5bb49 847 let rrd_prefix = format!("datastore/{}", config.name);
8c03041a 848 let path = std::path::Path::new(&config.path);
013fa7bb 849 gather_disk_stats(disk_manager.clone(), path, &rrd_prefix, save);
d0833a70
DM
850 }
851 }
852 Err(err) => {
853 eprintln!("read datastore config failed - {}", err);
854 }
855 }
856
4f951399 857 });
eaeda365 858}
dd15c0aa 859
b15751bf
DM
860fn check_schedule(worker_type: &str, event_str: &str, id: &str) -> bool {
861 let event = match parse_calendar_event(event_str) {
82c05b41
HL
862 Ok(event) => event,
863 Err(err) => {
864 eprintln!("unable to parse schedule '{}' - {}", event_str, err);
865 return false;
866 }
867 };
868
b15751bf 869 let last = match jobstate::last_run_time(worker_type, &id) {
82c05b41
HL
870 Ok(time) => time,
871 Err(err) => {
872 eprintln!("could not get last run time of {} {}: {}", worker_type, id, err);
873 return false;
874 }
875 };
876
877 let next = match compute_next_event(&event, last, false) {
878 Ok(Some(next)) => next,
879 Ok(None) => return false,
880 Err(err) => {
881 eprintln!("compute_next_event for '{}' failed - {}", event_str, err);
882 return false;
883 }
884 };
885
886 let now = proxmox::tools::time::epoch_i64();
887 next <= now
888}
889
013fa7bb 890fn gather_disk_stats(disk_manager: Arc<DiskManage>, path: &Path, rrd_prefix: &str, save: bool) {
91e5bb49 891
934f5bb8 892 match proxmox_backup::tools::disks::disk_usage(path) {
33070956 893 Ok(status) => {
91e5bb49 894 let rrd_key = format!("{}/total", rrd_prefix);
33070956 895 rrd_update_gauge(&rrd_key, status.total as f64, save);
91e5bb49 896 let rrd_key = format!("{}/used", rrd_prefix);
33070956 897 rrd_update_gauge(&rrd_key, status.used as f64, save);
91e5bb49
DM
898 }
899 Err(err) => {
900 eprintln!("read disk_usage on {:?} failed - {}", path, err);
901 }
902 }
903
934f5bb8
DM
904 match disk_manager.find_mounted_device(path) {
905 Ok(None) => {},
906 Ok(Some((fs_type, device, source))) => {
907 let mut device_stat = None;
908 match fs_type.as_str() {
909 "zfs" => {
368f4c54
DC
910 if let Some(source) = source {
911 let pool = get_pool_from_dataset(&source).unwrap_or(&source);
912 match zfs_pool_stats(pool) {
934f5bb8
DM
913 Ok(stat) => device_stat = stat,
914 Err(err) => eprintln!("zfs_pool_stats({:?}) failed - {}", pool, err),
91e5bb49
DM
915 }
916 }
934f5bb8
DM
917 }
918 _ => {
919 if let Ok(disk) = disk_manager.clone().disk_by_dev_num(device.into_dev_t()) {
920 match disk.read_stat() {
921 Ok(stat) => device_stat = stat,
922 Err(err) => eprintln!("disk.read_stat {:?} failed - {}", path, err),
91e5bb49
DM
923 }
924 }
925 }
91e5bb49 926 }
934f5bb8
DM
927 if let Some(stat) = device_stat {
928 let rrd_key = format!("{}/read_ios", rrd_prefix);
929 rrd_update_derive(&rrd_key, stat.read_ios as f64, save);
930 let rrd_key = format!("{}/read_bytes", rrd_prefix);
931 rrd_update_derive(&rrd_key, (stat.read_sectors*512) as f64, save);
dd15c0aa 932
934f5bb8
DM
933 let rrd_key = format!("{}/write_ios", rrd_prefix);
934 rrd_update_derive(&rrd_key, stat.write_ios as f64, save);
935 let rrd_key = format!("{}/write_bytes", rrd_prefix);
936 rrd_update_derive(&rrd_key, (stat.write_sectors*512) as f64, save);
dd15c0aa 937
934f5bb8
DM
938 let rrd_key = format!("{}/io_ticks", rrd_prefix);
939 rrd_update_derive(&rrd_key, (stat.io_ticks as f64)/1000.0, save);
8c03041a
DM
940 }
941 }
934f5bb8
DM
942 Err(err) => {
943 eprintln!("find_mounted_device failed - {}", err);
944 }
8c03041a 945 }
8c03041a 946}