1 use std
::collections
::{HashSet, HashMap}
;
3 use std
::os
::unix
::ffi
::OsStrExt
;
4 use std
::sync
::{Arc, Mutex}
;
6 use anyhow
::{bail, format_err, Error}
;
8 use hyper
::http
::request
::Parts
;
9 use hyper
::{header, Body, Response, StatusCode}
;
10 use serde_json
::{json, Value}
;
13 api
, ApiResponseFuture
, ApiHandler
, ApiMethod
, Router
,
14 RpcEnvironment
, RpcEnvironmentType
, Permission
16 use proxmox
::api
::router
::SubdirMap
;
17 use proxmox
::api
::schema
::*;
18 use proxmox
::tools
::fs
::{replace_file, CreateOptions}
;
19 use proxmox
::try_block
;
20 use proxmox
::{http_err, identity, list_subdirs_api_method, sortable}
;
22 use pxar
::accessor
::aio
::Accessor
;
25 use crate::api2
::types
::*;
26 use crate::api2
::node
::rrd
::create_value_from_rrd
;
28 use crate::config
::datastore
;
29 use crate::config
::cached_user_info
::CachedUserInfo
;
31 use crate::server
::WorkerTask
;
32 use crate::tools
::{self, AsyncReaderStream, WrappedReaderStream}
;
33 use crate::config
::acl
::{
35 PRIV_DATASTORE_MODIFY
,
38 PRIV_DATASTORE_BACKUP
,
41 fn check_backup_owner(
45 ) -> Result
<(), Error
> {
46 let owner
= store
.get_owner(group
)?
;
48 bail
!("backup owner check failed ({} != {})", userid
, owner
);
55 backup_dir
: &BackupDir
,
56 ) -> Result
<(BackupManifest
, Vec
<BackupContent
>), Error
> {
58 let (manifest
, index_size
) = store
.load_manifest(backup_dir
)?
;
60 let mut result
= Vec
::new();
61 for item
in manifest
.files() {
62 result
.push(BackupContent
{
63 filename
: item
.filename
.clone(),
64 crypt_mode
: Some(item
.crypt_mode
),
65 size
: Some(item
.size
),
69 result
.push(BackupContent
{
70 filename
: MANIFEST_BLOB_NAME
.to_string(),
71 crypt_mode
: match manifest
.signature
{
72 Some(_
) => Some(CryptMode
::SignOnly
),
73 None
=> Some(CryptMode
::None
),
75 size
: Some(index_size
),
78 Ok((manifest
, result
))
81 fn get_all_snapshot_files(
84 ) -> Result
<(BackupManifest
, Vec
<BackupContent
>), Error
> {
86 let (manifest
, mut files
) = read_backup_index(&store
, &info
.backup_dir
)?
;
88 let file_set
= files
.iter().fold(HashSet
::new(), |mut acc
, item
| {
89 acc
.insert(item
.filename
.clone());
93 for file
in &info
.files
{
94 if file_set
.contains(file
) { continue; }
95 files
.push(BackupContent
{
96 filename
: file
.to_string(),
102 Ok((manifest
, files
))
105 fn group_backups(backup_list
: Vec
<BackupInfo
>) -> HashMap
<String
, Vec
<BackupInfo
>> {
107 let mut group_hash
= HashMap
::new();
109 for info
in backup_list
{
110 let group_id
= info
.backup_dir
.group().group_path().to_str().unwrap().to_owned();
111 let time_list
= group_hash
.entry(group_id
).or_insert(vec
![]);
112 time_list
.push(info
);
122 schema
: DATASTORE_SCHEMA
,
128 description
: "Returns the list of backup groups.",
134 permission
: &Permission
::Privilege(
135 &["datastore", "{store}"],
136 PRIV_DATASTORE_AUDIT
| PRIV_DATASTORE_BACKUP
,
140 /// List backup groups.
143 rpcenv
: &mut dyn RpcEnvironment
,
144 ) -> Result
<Vec
<GroupListItem
>, Error
> {
146 let userid
: Userid
= rpcenv
.get_user().unwrap().parse()?
;
147 let user_info
= CachedUserInfo
::new()?
;
148 let user_privs
= user_info
.lookup_privs(&userid
, &["datastore", &store
]);
150 let datastore
= DataStore
::lookup_datastore(&store
)?
;
152 let backup_list
= BackupInfo
::list_backups(&datastore
.base_path())?
;
154 let group_hash
= group_backups(backup_list
);
156 let mut groups
= Vec
::new();
158 for (_group_id
, mut list
) in group_hash
{
160 BackupInfo
::sort_list(&mut list
, false);
164 let group
= info
.backup_dir
.group();
166 let list_all
= (user_privs
& PRIV_DATASTORE_AUDIT
) != 0;
167 let owner
= datastore
.get_owner(group
)?
;
169 if owner
!= userid { continue; }
172 let result_item
= GroupListItem
{
173 backup_type
: group
.backup_type().to_string(),
174 backup_id
: group
.backup_id().to_string(),
175 last_backup
: info
.backup_dir
.backup_time(),
176 backup_count
: list
.len() as u64,
177 files
: info
.files
.clone(),
180 groups
.push(result_item
);
190 schema
: DATASTORE_SCHEMA
,
193 schema
: BACKUP_TYPE_SCHEMA
,
196 schema
: BACKUP_ID_SCHEMA
,
199 schema
: BACKUP_TIME_SCHEMA
,
205 description
: "Returns the list of archive files inside a backup snapshots.",
211 permission
: &Permission
::Privilege(
212 &["datastore", "{store}"],
213 PRIV_DATASTORE_AUDIT
| PRIV_DATASTORE_READ
| PRIV_DATASTORE_BACKUP
,
217 /// List snapshot files.
218 pub fn list_snapshot_files(
224 rpcenv
: &mut dyn RpcEnvironment
,
225 ) -> Result
<Vec
<BackupContent
>, Error
> {
227 let userid
: Userid
= rpcenv
.get_user().unwrap().parse()?
;
228 let user_info
= CachedUserInfo
::new()?
;
229 let user_privs
= user_info
.lookup_privs(&userid
, &["datastore", &store
]);
231 let datastore
= DataStore
::lookup_datastore(&store
)?
;
233 let snapshot
= BackupDir
::new(backup_type
, backup_id
, backup_time
)?
;
235 let allowed
= (user_privs
& (PRIV_DATASTORE_AUDIT
| PRIV_DATASTORE_READ
)) != 0;
236 if !allowed { check_backup_owner(&datastore, snapshot.group(), &userid)?; }
238 let info
= BackupInfo
::new(&datastore
.base_path(), snapshot
)?
;
240 let (_manifest
, files
) = get_all_snapshot_files(&datastore
, &info
)?
;
249 schema
: DATASTORE_SCHEMA
,
252 schema
: BACKUP_TYPE_SCHEMA
,
255 schema
: BACKUP_ID_SCHEMA
,
258 schema
: BACKUP_TIME_SCHEMA
,
263 permission
: &Permission
::Privilege(
264 &["datastore", "{store}"],
265 PRIV_DATASTORE_MODIFY
| PRIV_DATASTORE_PRUNE
,
269 /// Delete backup snapshot.
276 rpcenv
: &mut dyn RpcEnvironment
,
277 ) -> Result
<Value
, Error
> {
279 let userid
: Userid
= rpcenv
.get_user().unwrap().parse()?
;
280 let user_info
= CachedUserInfo
::new()?
;
281 let user_privs
= user_info
.lookup_privs(&userid
, &["datastore", &store
]);
283 let snapshot
= BackupDir
::new(backup_type
, backup_id
, backup_time
)?
;
285 let datastore
= DataStore
::lookup_datastore(&store
)?
;
287 let allowed
= (user_privs
& PRIV_DATASTORE_MODIFY
) != 0;
288 if !allowed { check_backup_owner(&datastore, snapshot.group(), &userid)?; }
290 datastore
.remove_backup_dir(&snapshot
, false)?
;
299 schema
: DATASTORE_SCHEMA
,
303 schema
: BACKUP_TYPE_SCHEMA
,
307 schema
: BACKUP_ID_SCHEMA
,
313 description
: "Returns the list of snapshots.",
315 type: SnapshotListItem
,
319 permission
: &Permission
::Privilege(
320 &["datastore", "{store}"],
321 PRIV_DATASTORE_AUDIT
| PRIV_DATASTORE_BACKUP
,
325 /// List backup snapshots.
326 pub fn list_snapshots (
328 backup_type
: Option
<String
>,
329 backup_id
: Option
<String
>,
332 rpcenv
: &mut dyn RpcEnvironment
,
333 ) -> Result
<Vec
<SnapshotListItem
>, Error
> {
335 let userid
: Userid
= rpcenv
.get_user().unwrap().parse()?
;
336 let user_info
= CachedUserInfo
::new()?
;
337 let user_privs
= user_info
.lookup_privs(&userid
, &["datastore", &store
]);
339 let datastore
= DataStore
::lookup_datastore(&store
)?
;
341 let base_path
= datastore
.base_path();
343 let backup_list
= BackupInfo
::list_backups(&base_path
)?
;
345 let mut snapshots
= vec
![];
347 for info
in backup_list
{
348 let group
= info
.backup_dir
.group();
349 if let Some(ref backup_type
) = backup_type
{
350 if backup_type
!= group
.backup_type() { continue; }
352 if let Some(ref backup_id
) = backup_id
{
353 if backup_id
!= group
.backup_id() { continue; }
356 let list_all
= (user_privs
& PRIV_DATASTORE_AUDIT
) != 0;
357 let owner
= datastore
.get_owner(group
)?
;
360 if owner
!= userid { continue; }
365 let (comment
, verification
, files
) = match get_all_snapshot_files(&datastore
, &info
) {
366 Ok((manifest
, files
)) => {
367 size
= Some(files
.iter().map(|x
| x
.size
.unwrap_or(0)).sum());
368 // extract the first line from notes
369 let comment
: Option
<String
> = manifest
.unprotected
["notes"]
371 .and_then(|notes
| notes
.lines().next())
374 let verify
= manifest
.unprotected
["verify_state"].clone();
375 let verify
: Option
<SnapshotVerifyState
> = match serde_json
::from_value(verify
) {
376 Ok(verify
) => verify
,
378 eprintln
!("error parsing verification state : '{}'", err
);
383 (comment
, verify
, files
)
386 eprintln
!("error during snapshot file listing: '{}'", err
);
393 .map(|x
| BackupContent
{
394 filename
: x
.to_string(),
403 let result_item
= SnapshotListItem
{
404 backup_type
: group
.backup_type().to_string(),
405 backup_id
: group
.backup_id().to_string(),
406 backup_time
: info
.backup_dir
.backup_time(),
414 snapshots
.push(result_item
);
424 schema
: DATASTORE_SCHEMA
,
432 permission
: &Permission
::Privilege(&["datastore", "{store}"], PRIV_DATASTORE_AUDIT
| PRIV_DATASTORE_BACKUP
, true),
435 /// Get datastore status.
439 _rpcenv
: &mut dyn RpcEnvironment
,
440 ) -> Result
<StorageStatus
, Error
> {
441 let datastore
= DataStore
::lookup_datastore(&store
)?
;
442 crate::tools
::disks
::disk_usage(&datastore
.base_path())
449 schema
: DATASTORE_SCHEMA
,
452 schema
: BACKUP_TYPE_SCHEMA
,
456 schema
: BACKUP_ID_SCHEMA
,
460 schema
: BACKUP_TIME_SCHEMA
,
469 permission
: &Permission
::Privilege(&["datastore", "{store}"], PRIV_DATASTORE_READ
| PRIV_DATASTORE_BACKUP
, true), // fixme
474 /// This function can verify a single backup snapshot, all backup from a backup group,
475 /// or all backups in the datastore.
478 backup_type
: Option
<String
>,
479 backup_id
: Option
<String
>,
480 backup_time
: Option
<i64>,
481 rpcenv
: &mut dyn RpcEnvironment
,
482 ) -> Result
<Value
, Error
> {
483 let datastore
= DataStore
::lookup_datastore(&store
)?
;
487 let mut backup_dir
= None
;
488 let mut backup_group
= None
;
490 match (backup_type
, backup_id
, backup_time
) {
491 (Some(backup_type
), Some(backup_id
), Some(backup_time
)) => {
492 worker_id
= format
!("{}_{}_{}_{:08X}", store
, backup_type
, backup_id
, backup_time
);
493 let dir
= BackupDir
::new(backup_type
, backup_id
, backup_time
)?
;
494 backup_dir
= Some(dir
);
496 (Some(backup_type
), Some(backup_id
), None
) => {
497 worker_id
= format
!("{}_{}_{}", store
, backup_type
, backup_id
);
498 let group
= BackupGroup
::new(backup_type
, backup_id
);
499 backup_group
= Some(group
);
501 (None
, None
, None
) => {
502 worker_id
= store
.clone();
504 _
=> bail
!("parameters do not specify a backup group or snapshot"),
507 let userid
: Userid
= rpcenv
.get_user().unwrap().parse()?
;
508 let to_stdout
= if rpcenv
.env_type() == RpcEnvironmentType
::CLI { true }
else { false }
;
510 let upid_str
= WorkerTask
::new_thread(
512 Some(worker_id
.clone()),
516 let verified_chunks
= Arc
::new(Mutex
::new(HashSet
::with_capacity(1024*16)));
517 let corrupt_chunks
= Arc
::new(Mutex
::new(HashSet
::with_capacity(64)));
519 let failed_dirs
= if let Some(backup_dir
) = backup_dir
{
520 let mut res
= Vec
::new();
521 if !verify_backup_dir(
527 worker
.upid().clone(),
529 res
.push(backup_dir
.to_string());
532 } else if let Some(backup_group
) = backup_group
{
533 let (_count
, failed_dirs
) = verify_backup_group(
544 verify_all_backups(datastore
, worker
.clone(), worker
.upid())?
546 if failed_dirs
.len() > 0 {
547 worker
.log("Failed to verify following snapshots:");
548 for dir
in failed_dirs
{
549 worker
.log(format
!("\t{}", dir
));
551 bail
!("verification failed - please check the log for details");
561 macro_rules
! add_common_prune_prameters
{
562 ( [ $
( $list1
:tt
)* ] ) => {
563 add_common_prune_prameters
!([$
( $list1
)* ] , [])
565 ( [ $
( $list1
:tt
)* ] , [ $
( $list2
:tt
)* ] ) => {
571 &PRUNE_SCHEMA_KEEP_DAILY
,
576 &PRUNE_SCHEMA_KEEP_HOURLY
,
581 &PRUNE_SCHEMA_KEEP_LAST
,
586 &PRUNE_SCHEMA_KEEP_MONTHLY
,
591 &PRUNE_SCHEMA_KEEP_WEEKLY
,
596 &PRUNE_SCHEMA_KEEP_YEARLY
,
603 pub const API_RETURN_SCHEMA_PRUNE
: Schema
= ArraySchema
::new(
604 "Returns the list of snapshots and a flag indicating if there are kept or removed.",
605 &PruneListItem
::API_SCHEMA
608 const API_METHOD_PRUNE
: ApiMethod
= ApiMethod
::new(
609 &ApiHandler
::Sync(&prune
),
611 "Prune the datastore.",
612 &add_common_prune_prameters
!([
613 ("backup-id", false, &BACKUP_ID_SCHEMA
),
614 ("backup-type", false, &BACKUP_TYPE_SCHEMA
),
615 ("dry-run", true, &BooleanSchema
::new(
616 "Just show what prune would do, but do not delete anything.")
620 ("store", false, &DATASTORE_SCHEMA
),
623 .returns(&API_RETURN_SCHEMA_PRUNE
)
624 .access(None
, &Permission
::Privilege(
625 &["datastore", "{store}"],
626 PRIV_DATASTORE_MODIFY
| PRIV_DATASTORE_PRUNE
,
633 rpcenv
: &mut dyn RpcEnvironment
,
634 ) -> Result
<Value
, Error
> {
636 let store
= tools
::required_string_param(¶m
, "store")?
;
637 let backup_type
= tools
::required_string_param(¶m
, "backup-type")?
;
638 let backup_id
= tools
::required_string_param(¶m
, "backup-id")?
;
640 let userid
: Userid
= rpcenv
.get_user().unwrap().parse()?
;
641 let user_info
= CachedUserInfo
::new()?
;
642 let user_privs
= user_info
.lookup_privs(&userid
, &["datastore", &store
]);
644 let dry_run
= param
["dry-run"].as_bool().unwrap_or(false);
646 let group
= BackupGroup
::new(backup_type
, backup_id
);
648 let datastore
= DataStore
::lookup_datastore(&store
)?
;
650 let allowed
= (user_privs
& PRIV_DATASTORE_MODIFY
) != 0;
651 if !allowed { check_backup_owner(&datastore, &group, &userid)?; }
653 let prune_options
= PruneOptions
{
654 keep_last
: param
["keep-last"].as_u64(),
655 keep_hourly
: param
["keep-hourly"].as_u64(),
656 keep_daily
: param
["keep-daily"].as_u64(),
657 keep_weekly
: param
["keep-weekly"].as_u64(),
658 keep_monthly
: param
["keep-monthly"].as_u64(),
659 keep_yearly
: param
["keep-yearly"].as_u64(),
662 let worker_id
= format
!("{}_{}_{}", store
, backup_type
, backup_id
);
664 let mut prune_result
= Vec
::new();
666 let list
= group
.list_backups(&datastore
.base_path())?
;
668 let mut prune_info
= compute_prune_info(list
, &prune_options
)?
;
670 prune_info
.reverse(); // delete older snapshots first
672 let keep_all
= !prune_options
.keeps_something();
675 for (info
, mut keep
) in prune_info
{
676 if keep_all { keep = true; }
678 let backup_time
= info
.backup_dir
.backup_time();
679 let group
= info
.backup_dir
.group();
681 prune_result
.push(json
!({
682 "backup-type": group
.backup_type(),
683 "backup-id": group
.backup_id(),
684 "backup-time": backup_time
,
688 return Ok(json
!(prune_result
));
692 // We use a WorkerTask just to have a task log, but run synchrounously
693 let worker
= WorkerTask
::new("prune", Some(worker_id
), Userid
::root_userid().clone(), true)?
;
695 let result
= try_block
! {
697 worker
.log("No prune selection - keeping all files.");
699 worker
.log(format
!("retention options: {}", prune_options
.cli_options_string()));
700 worker
.log(format
!("Starting prune on store \"{}\" group \"{}/{}\"",
701 store
, backup_type
, backup_id
));
704 for (info
, mut keep
) in prune_info
{
705 if keep_all { keep = true; }
707 let backup_time
= info
.backup_dir
.backup_time();
708 let timestamp
= info
.backup_dir
.backup_time_string();
709 let group
= info
.backup_dir
.group();
717 if keep { "keep" }
else { "remove" }
,
722 prune_result
.push(json
!({
723 "backup-type": group
.backup_type(),
724 "backup-id": group
.backup_id(),
725 "backup-time": backup_time
,
729 if !(dry_run
|| keep
) {
730 datastore
.remove_backup_dir(&info
.backup_dir
, true)?
;
737 worker
.log_result(&result
);
739 if let Err(err
) = result
{
740 bail
!("prune failed - {}", err
);
743 Ok(json
!(prune_result
))
750 schema
: DATASTORE_SCHEMA
,
758 permission
: &Permission
::Privilege(&["datastore", "{store}"], PRIV_DATASTORE_MODIFY
, false),
761 /// Start garbage collection.
762 fn start_garbage_collection(
765 rpcenv
: &mut dyn RpcEnvironment
,
766 ) -> Result
<Value
, Error
> {
768 let datastore
= DataStore
::lookup_datastore(&store
)?
;
770 println
!("Starting garbage collection on store {}", store
);
772 let to_stdout
= if rpcenv
.env_type() == RpcEnvironmentType
::CLI { true }
else { false }
;
774 let upid_str
= WorkerTask
::new_thread(
775 "garbage_collection",
777 Userid
::root_userid().clone(),
780 worker
.log(format
!("starting garbage collection on store {}", store
));
781 datastore
.garbage_collection(&*worker
, worker
.upid())
792 schema
: DATASTORE_SCHEMA
,
797 type: GarbageCollectionStatus
,
800 permission
: &Permission
::Privilege(&["datastore", "{store}"], PRIV_DATASTORE_AUDIT
, false),
803 /// Garbage collection status.
804 pub fn garbage_collection_status(
807 _rpcenv
: &mut dyn RpcEnvironment
,
808 ) -> Result
<GarbageCollectionStatus
, Error
> {
810 let datastore
= DataStore
::lookup_datastore(&store
)?
;
812 let status
= datastore
.last_gc_status();
819 description
: "List the accessible datastores.",
822 description
: "Datastore name and description.",
825 schema
: DATASTORE_SCHEMA
,
829 schema
: SINGLE_LINE_COMMENT_SCHEMA
,
835 permission
: &Permission
::Anybody
,
839 fn get_datastore_list(
842 rpcenv
: &mut dyn RpcEnvironment
,
843 ) -> Result
<Value
, Error
> {
845 let (config
, _digest
) = datastore
::config()?
;
847 let userid
: Userid
= rpcenv
.get_user().unwrap().parse()?
;
848 let user_info
= CachedUserInfo
::new()?
;
850 let mut list
= Vec
::new();
852 for (store
, (_
, data
)) in &config
.sections
{
853 let user_privs
= user_info
.lookup_privs(&userid
, &["datastore", &store
]);
854 let allowed
= (user_privs
& (PRIV_DATASTORE_AUDIT
| PRIV_DATASTORE_BACKUP
)) != 0;
856 let mut entry
= json
!({ "store": store }
);
857 if let Some(comment
) = data
["comment"].as_str() {
858 entry
["comment"] = comment
.into();
868 pub const API_METHOD_DOWNLOAD_FILE
: ApiMethod
= ApiMethod
::new(
869 &ApiHandler
::AsyncHttp(&download_file
),
871 "Download single raw file from backup snapshot.",
873 ("store", false, &DATASTORE_SCHEMA
),
874 ("backup-type", false, &BACKUP_TYPE_SCHEMA
),
875 ("backup-id", false, &BACKUP_ID_SCHEMA
),
876 ("backup-time", false, &BACKUP_TIME_SCHEMA
),
877 ("file-name", false, &BACKUP_ARCHIVE_NAME_SCHEMA
),
880 ).access(None
, &Permission
::Privilege(
881 &["datastore", "{store}"],
882 PRIV_DATASTORE_READ
| PRIV_DATASTORE_BACKUP
,
891 rpcenv
: Box
<dyn RpcEnvironment
>,
892 ) -> ApiResponseFuture
{
895 let store
= tools
::required_string_param(¶m
, "store")?
;
896 let datastore
= DataStore
::lookup_datastore(store
)?
;
898 let userid
: Userid
= rpcenv
.get_user().unwrap().parse()?
;
899 let user_info
= CachedUserInfo
::new()?
;
900 let user_privs
= user_info
.lookup_privs(&userid
, &["datastore", &store
]);
902 let file_name
= tools
::required_string_param(¶m
, "file-name")?
.to_owned();
904 let backup_type
= tools
::required_string_param(¶m
, "backup-type")?
;
905 let backup_id
= tools
::required_string_param(¶m
, "backup-id")?
;
906 let backup_time
= tools
::required_integer_param(¶m
, "backup-time")?
;
908 let backup_dir
= BackupDir
::new(backup_type
, backup_id
, backup_time
)?
;
910 let allowed
= (user_privs
& PRIV_DATASTORE_READ
) != 0;
911 if !allowed { check_backup_owner(&datastore, backup_dir.group(), &userid)?; }
913 println
!("Download {} from {} ({}/{})", file_name
, store
, backup_dir
, file_name
);
915 let mut path
= datastore
.base_path();
916 path
.push(backup_dir
.relative_path());
917 path
.push(&file_name
);
919 let file
= tokio
::fs
::File
::open(&path
)
921 .map_err(|err
| http_err
!(BAD_REQUEST
, "File open failed: {}", err
))?
;
923 let payload
= tokio_util
::codec
::FramedRead
::new(file
, tokio_util
::codec
::BytesCodec
::new())
924 .map_ok(|bytes
| hyper
::body
::Bytes
::from(bytes
.freeze()))
925 .map_err(move |err
| {
926 eprintln
!("error during streaming of '{:?}' - {}", &path
, err
);
929 let body
= Body
::wrap_stream(payload
);
931 // fixme: set other headers ?
932 Ok(Response
::builder()
933 .status(StatusCode
::OK
)
934 .header(header
::CONTENT_TYPE
, "application/octet-stream")
941 pub const API_METHOD_DOWNLOAD_FILE_DECODED
: ApiMethod
= ApiMethod
::new(
942 &ApiHandler
::AsyncHttp(&download_file_decoded
),
944 "Download single decoded file from backup snapshot. Only works if it's not encrypted.",
946 ("store", false, &DATASTORE_SCHEMA
),
947 ("backup-type", false, &BACKUP_TYPE_SCHEMA
),
948 ("backup-id", false, &BACKUP_ID_SCHEMA
),
949 ("backup-time", false, &BACKUP_TIME_SCHEMA
),
950 ("file-name", false, &BACKUP_ARCHIVE_NAME_SCHEMA
),
953 ).access(None
, &Permission
::Privilege(
954 &["datastore", "{store}"],
955 PRIV_DATASTORE_READ
| PRIV_DATASTORE_BACKUP
,
959 fn download_file_decoded(
964 rpcenv
: Box
<dyn RpcEnvironment
>,
965 ) -> ApiResponseFuture
{
968 let store
= tools
::required_string_param(¶m
, "store")?
;
969 let datastore
= DataStore
::lookup_datastore(store
)?
;
971 let userid
: Userid
= rpcenv
.get_user().unwrap().parse()?
;
972 let user_info
= CachedUserInfo
::new()?
;
973 let user_privs
= user_info
.lookup_privs(&userid
, &["datastore", &store
]);
975 let file_name
= tools
::required_string_param(¶m
, "file-name")?
.to_owned();
977 let backup_type
= tools
::required_string_param(¶m
, "backup-type")?
;
978 let backup_id
= tools
::required_string_param(¶m
, "backup-id")?
;
979 let backup_time
= tools
::required_integer_param(¶m
, "backup-time")?
;
981 let backup_dir
= BackupDir
::new(backup_type
, backup_id
, backup_time
)?
;
983 let allowed
= (user_privs
& PRIV_DATASTORE_READ
) != 0;
984 if !allowed { check_backup_owner(&datastore, backup_dir.group(), &userid)?; }
986 let (manifest
, files
) = read_backup_index(&datastore
, &backup_dir
)?
;
988 if file
.filename
== file_name
&& file
.crypt_mode
== Some(CryptMode
::Encrypt
) {
989 bail
!("cannot decode '{}' - is encrypted", file_name
);
993 println
!("Download {} from {} ({}/{})", file_name
, store
, backup_dir
, file_name
);
995 let mut path
= datastore
.base_path();
996 path
.push(backup_dir
.relative_path());
997 path
.push(&file_name
);
999 let extension
= file_name
.rsplitn(2, '
.'
).next().unwrap();
1001 let body
= match extension
{
1003 let index
= DynamicIndexReader
::open(&path
)
1004 .map_err(|err
| format_err
!("unable to read dynamic index '{:?}' - {}", &path
, err
))?
;
1005 let (csum
, size
) = index
.compute_csum();
1006 manifest
.verify_file(&file_name
, &csum
, size
)?
;
1008 let chunk_reader
= LocalChunkReader
::new(datastore
, None
, CryptMode
::None
);
1009 let reader
= AsyncIndexReader
::new(index
, chunk_reader
);
1010 Body
::wrap_stream(AsyncReaderStream
::new(reader
)
1011 .map_err(move |err
| {
1012 eprintln
!("error during streaming of '{:?}' - {}", path
, err
);
1017 let index
= FixedIndexReader
::open(&path
)
1018 .map_err(|err
| format_err
!("unable to read fixed index '{:?}' - {}", &path
, err
))?
;
1020 let (csum
, size
) = index
.compute_csum();
1021 manifest
.verify_file(&file_name
, &csum
, size
)?
;
1023 let chunk_reader
= LocalChunkReader
::new(datastore
, None
, CryptMode
::None
);
1024 let reader
= AsyncIndexReader
::new(index
, chunk_reader
);
1025 Body
::wrap_stream(AsyncReaderStream
::with_buffer_size(reader
, 4*1024*1024)
1026 .map_err(move |err
| {
1027 eprintln
!("error during streaming of '{:?}' - {}", path
, err
);
1032 let file
= std
::fs
::File
::open(&path
)
1033 .map_err(|err
| http_err
!(BAD_REQUEST
, "File open failed: {}", err
))?
;
1035 // FIXME: load full blob to verify index checksum?
1038 WrappedReaderStream
::new(DataBlobReader
::new(file
, None
)?
)
1039 .map_err(move |err
| {
1040 eprintln
!("error during streaming of '{:?}' - {}", path
, err
);
1046 bail
!("cannot download '{}' files", extension
);
1050 // fixme: set other headers ?
1051 Ok(Response
::builder()
1052 .status(StatusCode
::OK
)
1053 .header(header
::CONTENT_TYPE
, "application/octet-stream")
1060 pub const API_METHOD_UPLOAD_BACKUP_LOG
: ApiMethod
= ApiMethod
::new(
1061 &ApiHandler
::AsyncHttp(&upload_backup_log
),
1063 "Upload the client backup log file into a backup snapshot ('client.log.blob').",
1065 ("store", false, &DATASTORE_SCHEMA
),
1066 ("backup-type", false, &BACKUP_TYPE_SCHEMA
),
1067 ("backup-id", false, &BACKUP_ID_SCHEMA
),
1068 ("backup-time", false, &BACKUP_TIME_SCHEMA
),
1072 Some("Only the backup creator/owner is allowed to do this."),
1073 &Permission
::Privilege(&["datastore", "{store}"], PRIV_DATASTORE_BACKUP
, false)
1076 fn upload_backup_log(
1081 rpcenv
: Box
<dyn RpcEnvironment
>,
1082 ) -> ApiResponseFuture
{
1085 let store
= tools
::required_string_param(¶m
, "store")?
;
1086 let datastore
= DataStore
::lookup_datastore(store
)?
;
1088 let file_name
= CLIENT_LOG_BLOB_NAME
;
1090 let backup_type
= tools
::required_string_param(¶m
, "backup-type")?
;
1091 let backup_id
= tools
::required_string_param(¶m
, "backup-id")?
;
1092 let backup_time
= tools
::required_integer_param(¶m
, "backup-time")?
;
1094 let backup_dir
= BackupDir
::new(backup_type
, backup_id
, backup_time
)?
;
1096 let userid
: Userid
= rpcenv
.get_user().unwrap().parse()?
;
1097 check_backup_owner(&datastore
, backup_dir
.group(), &userid
)?
;
1099 let mut path
= datastore
.base_path();
1100 path
.push(backup_dir
.relative_path());
1101 path
.push(&file_name
);
1104 bail
!("backup already contains a log.");
1107 println
!("Upload backup log to {}/{}/{}/{}/{}", store
,
1108 backup_type
, backup_id
, backup_dir
.backup_time_string(), file_name
);
1111 .map_err(Error
::from
)
1112 .try_fold(Vec
::new(), |mut acc
, chunk
| {
1113 acc
.extend_from_slice(&*chunk
);
1114 future
::ok
::<_
, Error
>(acc
)
1118 // always verify blob/CRC at server side
1119 let blob
= DataBlob
::load_from_reader(&mut &data
[..])?
;
1121 replace_file(&path
, blob
.raw_data(), CreateOptions
::new())?
;
1123 // fixme: use correct formatter
1124 Ok(crate::server
::formatter
::json_response(Ok(Value
::Null
)))
1132 schema
: DATASTORE_SCHEMA
,
1135 schema
: BACKUP_TYPE_SCHEMA
,
1138 schema
: BACKUP_ID_SCHEMA
,
1141 schema
: BACKUP_TIME_SCHEMA
,
1144 description
: "Base64 encoded path.",
1150 permission
: &Permission
::Privilege(&["datastore", "{store}"], PRIV_DATASTORE_READ
| PRIV_DATASTORE_BACKUP
, true),
1153 /// Get the entries of the given path of the catalog
1156 backup_type
: String
,
1162 rpcenv
: &mut dyn RpcEnvironment
,
1163 ) -> Result
<Value
, Error
> {
1164 let datastore
= DataStore
::lookup_datastore(&store
)?
;
1166 let userid
: Userid
= rpcenv
.get_user().unwrap().parse()?
;
1167 let user_info
= CachedUserInfo
::new()?
;
1168 let user_privs
= user_info
.lookup_privs(&userid
, &["datastore", &store
]);
1170 let backup_dir
= BackupDir
::new(backup_type
, backup_id
, backup_time
)?
;
1172 let allowed
= (user_privs
& PRIV_DATASTORE_READ
) != 0;
1173 if !allowed { check_backup_owner(&datastore, backup_dir.group(), &userid)?; }
1175 let file_name
= CATALOG_NAME
;
1177 let (manifest
, files
) = read_backup_index(&datastore
, &backup_dir
)?
;
1179 if file
.filename
== file_name
&& file
.crypt_mode
== Some(CryptMode
::Encrypt
) {
1180 bail
!("cannot decode '{}' - is encrypted", file_name
);
1184 let mut path
= datastore
.base_path();
1185 path
.push(backup_dir
.relative_path());
1186 path
.push(file_name
);
1188 let index
= DynamicIndexReader
::open(&path
)
1189 .map_err(|err
| format_err
!("unable to read dynamic index '{:?}' - {}", &path
, err
))?
;
1191 let (csum
, size
) = index
.compute_csum();
1192 manifest
.verify_file(&file_name
, &csum
, size
)?
;
1194 let chunk_reader
= LocalChunkReader
::new(datastore
, None
, CryptMode
::None
);
1195 let reader
= BufferedDynamicReader
::new(index
, chunk_reader
);
1197 let mut catalog_reader
= CatalogReader
::new(reader
);
1198 let mut current
= catalog_reader
.root()?
;
1199 let mut components
= vec
![];
1202 if filepath
!= "root" {
1203 components
= base64
::decode(filepath
)?
;
1204 if components
.len() > 0 && components
[0] == '
/'
as u8 {
1205 components
.remove(0);
1207 for component
in components
.split(|c
| *c
== '
/'
as u8) {
1208 if let Some(entry
) = catalog_reader
.lookup(¤t
, component
)?
{
1211 bail
!("path {:?} not found in catalog", &String
::from_utf8_lossy(&components
));
1216 let mut res
= Vec
::new();
1218 for direntry
in catalog_reader
.read_dir(¤t
)?
{
1219 let mut components
= components
.clone();
1220 components
.push('
/'
as u8);
1221 components
.extend(&direntry
.name
);
1222 let path
= base64
::encode(components
);
1223 let text
= String
::from_utf8_lossy(&direntry
.name
);
1224 let mut entry
= json
!({
1227 "type": CatalogEntryType
::from(&direntry
.attr
).to_string(),
1230 match direntry
.attr
{
1231 DirEntryAttribute
::Directory { start: _ }
=> {
1232 entry
["leaf"] = false.into();
1234 DirEntryAttribute
::File { size, mtime }
=> {
1235 entry
["size"] = size
.into();
1236 entry
["mtime"] = mtime
.into();
1247 pub const API_METHOD_PXAR_FILE_DOWNLOAD
: ApiMethod
= ApiMethod
::new(
1248 &ApiHandler
::AsyncHttp(&pxar_file_download
),
1250 "Download single file from pxar file of a backup snapshot. Only works if it's not encrypted.",
1252 ("store", false, &DATASTORE_SCHEMA
),
1253 ("backup-type", false, &BACKUP_TYPE_SCHEMA
),
1254 ("backup-id", false, &BACKUP_ID_SCHEMA
),
1255 ("backup-time", false, &BACKUP_TIME_SCHEMA
),
1256 ("filepath", false, &StringSchema
::new("Base64 encoded path").schema()),
1259 ).access(None
, &Permission
::Privilege(
1260 &["datastore", "{store}"],
1261 PRIV_DATASTORE_READ
| PRIV_DATASTORE_BACKUP
,
1265 fn pxar_file_download(
1270 rpcenv
: Box
<dyn RpcEnvironment
>,
1271 ) -> ApiResponseFuture
{
1274 let store
= tools
::required_string_param(¶m
, "store")?
;
1275 let datastore
= DataStore
::lookup_datastore(&store
)?
;
1277 let userid
: Userid
= rpcenv
.get_user().unwrap().parse()?
;
1278 let user_info
= CachedUserInfo
::new()?
;
1279 let user_privs
= user_info
.lookup_privs(&userid
, &["datastore", &store
]);
1281 let filepath
= tools
::required_string_param(¶m
, "filepath")?
.to_owned();
1283 let backup_type
= tools
::required_string_param(¶m
, "backup-type")?
;
1284 let backup_id
= tools
::required_string_param(¶m
, "backup-id")?
;
1285 let backup_time
= tools
::required_integer_param(¶m
, "backup-time")?
;
1287 let backup_dir
= BackupDir
::new(backup_type
, backup_id
, backup_time
)?
;
1289 let allowed
= (user_privs
& PRIV_DATASTORE_READ
) != 0;
1290 if !allowed { check_backup_owner(&datastore, backup_dir.group(), &userid)?; }
1292 let mut components
= base64
::decode(&filepath
)?
;
1293 if components
.len() > 0 && components
[0] == '
/'
as u8 {
1294 components
.remove(0);
1297 let mut split
= components
.splitn(2, |c
| *c
== '
/'
as u8);
1298 let pxar_name
= std
::str::from_utf8(split
.next().unwrap())?
;
1299 let file_path
= split
.next().ok_or(format_err
!("filepath looks strange '{}'", filepath
))?
;
1300 let (manifest
, files
) = read_backup_index(&datastore
, &backup_dir
)?
;
1302 if file
.filename
== pxar_name
&& file
.crypt_mode
== Some(CryptMode
::Encrypt
) {
1303 bail
!("cannot decode '{}' - is encrypted", pxar_name
);
1307 let mut path
= datastore
.base_path();
1308 path
.push(backup_dir
.relative_path());
1309 path
.push(pxar_name
);
1311 let index
= DynamicIndexReader
::open(&path
)
1312 .map_err(|err
| format_err
!("unable to read dynamic index '{:?}' - {}", &path
, err
))?
;
1314 let (csum
, size
) = index
.compute_csum();
1315 manifest
.verify_file(&pxar_name
, &csum
, size
)?
;
1317 let chunk_reader
= LocalChunkReader
::new(datastore
, None
, CryptMode
::None
);
1318 let reader
= BufferedDynamicReader
::new(index
, chunk_reader
);
1319 let archive_size
= reader
.archive_size();
1320 let reader
= LocalDynamicReadAt
::new(reader
);
1322 let decoder
= Accessor
::new(reader
, archive_size
).await?
;
1323 let root
= decoder
.open_root().await?
;
1325 .lookup(OsStr
::from_bytes(file_path
)).await?
1326 .ok_or(format_err
!("error opening '{:?}'", file_path
))?
;
1328 let file
= match file
.kind() {
1329 EntryKind
::File { .. }
=> file
,
1330 EntryKind
::Hardlink(_
) => {
1331 decoder
.follow_hardlink(&file
).await?
1334 other
=> bail
!("cannot download file of type {:?}", other
),
1337 let body
= Body
::wrap_stream(
1338 AsyncReaderStream
::new(file
.contents().await?
)
1339 .map_err(move |err
| {
1340 eprintln
!("error during streaming of '{:?}' - {}", filepath
, err
);
1345 // fixme: set other headers ?
1346 Ok(Response
::builder()
1347 .status(StatusCode
::OK
)
1348 .header(header
::CONTENT_TYPE
, "application/octet-stream")
1358 schema
: DATASTORE_SCHEMA
,
1361 type: RRDTimeFrameResolution
,
1369 permission
: &Permission
::Privilege(&["datastore", "{store}"], PRIV_DATASTORE_AUDIT
| PRIV_DATASTORE_BACKUP
, true),
1372 /// Read datastore stats
1375 timeframe
: RRDTimeFrameResolution
,
1378 ) -> Result
<Value
, Error
> {
1380 create_value_from_rrd(
1381 &format
!("datastore/{}", store
),
1384 "read_ios", "read_bytes",
1385 "write_ios", "write_bytes",
1397 schema
: DATASTORE_SCHEMA
,
1400 schema
: BACKUP_TYPE_SCHEMA
,
1403 schema
: BACKUP_ID_SCHEMA
,
1406 schema
: BACKUP_TIME_SCHEMA
,
1411 permission
: &Permission
::Privilege(&["datastore", "{store}"], PRIV_DATASTORE_READ
| PRIV_DATASTORE_BACKUP
, true),
1414 /// Get "notes" for a specific backup
1417 backup_type
: String
,
1420 rpcenv
: &mut dyn RpcEnvironment
,
1421 ) -> Result
<String
, Error
> {
1422 let datastore
= DataStore
::lookup_datastore(&store
)?
;
1424 let userid
: Userid
= rpcenv
.get_user().unwrap().parse()?
;
1425 let user_info
= CachedUserInfo
::new()?
;
1426 let user_privs
= user_info
.lookup_privs(&userid
, &["datastore", &store
]);
1428 let backup_dir
= BackupDir
::new(backup_type
, backup_id
, backup_time
)?
;
1430 let allowed
= (user_privs
& PRIV_DATASTORE_READ
) != 0;
1431 if !allowed { check_backup_owner(&datastore, backup_dir.group(), &userid)?; }
1433 let manifest
= datastore
.load_manifest_json(&backup_dir
)?
;
1435 let notes
= manifest
["unprotected"]["notes"]
1439 Ok(String
::from(notes
))
1446 schema
: DATASTORE_SCHEMA
,
1449 schema
: BACKUP_TYPE_SCHEMA
,
1452 schema
: BACKUP_ID_SCHEMA
,
1455 schema
: BACKUP_TIME_SCHEMA
,
1458 description
: "A multiline text.",
1463 permission
: &Permission
::Privilege(&["datastore", "{store}"], PRIV_DATASTORE_MODIFY
, true),
1466 /// Set "notes" for a specific backup
1469 backup_type
: String
,
1473 rpcenv
: &mut dyn RpcEnvironment
,
1474 ) -> Result
<(), Error
> {
1475 let datastore
= DataStore
::lookup_datastore(&store
)?
;
1477 let userid
: Userid
= rpcenv
.get_user().unwrap().parse()?
;
1478 let user_info
= CachedUserInfo
::new()?
;
1479 let user_privs
= user_info
.lookup_privs(&userid
, &["datastore", &store
]);
1481 let backup_dir
= BackupDir
::new(backup_type
, backup_id
, backup_time
)?
;
1483 let allowed
= (user_privs
& PRIV_DATASTORE_READ
) != 0;
1484 if !allowed { check_backup_owner(&datastore, backup_dir.group(), &userid)?; }
1486 let mut manifest
= datastore
.load_manifest_json(&backup_dir
)?
;
1488 manifest
["unprotected"]["notes"] = notes
.into();
1490 datastore
.store_manifest(&backup_dir
, manifest
)?
;
1499 schema
: DATASTORE_SCHEMA
,
1502 schema
: BACKUP_TYPE_SCHEMA
,
1505 schema
: BACKUP_ID_SCHEMA
,
1513 permission
: &Permission
::Privilege(&["datastore", "{store}"], PRIV_DATASTORE_MODIFY
, true),
1516 /// Change owner of a backup group
1517 fn set_backup_owner(
1519 backup_type
: String
,
1522 rpcenv
: &mut dyn RpcEnvironment
,
1523 ) -> Result
<(), Error
> {
1525 let datastore
= DataStore
::lookup_datastore(&store
)?
;
1527 let backup_group
= BackupGroup
::new(backup_type
, backup_id
);
1529 let user_info
= CachedUserInfo
::new()?
;
1531 if !user_info
.is_active_user(&new_owner
) {
1532 bail
!("user '{}' is inactive or non-existent", new_owner
);
1535 datastore
.set_owner(&backup_group
, &new_owner
, true)?
;
1541 const DATASTORE_INFO_SUBDIRS
: SubdirMap
= &[
1545 .get(&API_METHOD_CATALOG
)
1550 .post(&API_METHOD_SET_BACKUP_OWNER
)
1555 .download(&API_METHOD_DOWNLOAD_FILE
)
1560 .download(&API_METHOD_DOWNLOAD_FILE_DECODED
)
1565 .get(&API_METHOD_LIST_SNAPSHOT_FILES
)
1570 .get(&API_METHOD_GARBAGE_COLLECTION_STATUS
)
1571 .post(&API_METHOD_START_GARBAGE_COLLECTION
)
1576 .get(&API_METHOD_LIST_GROUPS
)
1581 .get(&API_METHOD_GET_NOTES
)
1582 .put(&API_METHOD_SET_NOTES
)
1587 .post(&API_METHOD_PRUNE
)
1590 "pxar-file-download",
1592 .download(&API_METHOD_PXAR_FILE_DOWNLOAD
)
1597 .get(&API_METHOD_GET_RRD_STATS
)
1602 .get(&API_METHOD_LIST_SNAPSHOTS
)
1603 .delete(&API_METHOD_DELETE_SNAPSHOT
)
1608 .get(&API_METHOD_STATUS
)
1611 "upload-backup-log",
1613 .upload(&API_METHOD_UPLOAD_BACKUP_LOG
)
1618 .post(&API_METHOD_VERIFY
)
1622 const DATASTORE_INFO_ROUTER
: Router
= Router
::new()
1623 .get(&list_subdirs_api_method
!(DATASTORE_INFO_SUBDIRS
))
1624 .subdirs(DATASTORE_INFO_SUBDIRS
);
1627 pub const ROUTER
: Router
= Router
::new()
1628 .get(&API_METHOD_GET_DATASTORE_LIST
)
1629 .match_all("store", &DATASTORE_INFO_ROUTER
);