1 use std
::collections
::{HashSet, HashMap}
;
3 use std
::os
::unix
::ffi
::OsStrExt
;
5 use anyhow
::{bail, format_err, Error}
;
7 use hyper
::http
::request
::Parts
;
8 use hyper
::{header, Body, Response, StatusCode}
;
9 use serde_json
::{json, Value}
;
12 api
, ApiResponseFuture
, ApiHandler
, ApiMethod
, Router
,
13 RpcEnvironment
, RpcEnvironmentType
, Permission
, UserInformation
};
14 use proxmox
::api
::router
::SubdirMap
;
15 use proxmox
::api
::schema
::*;
16 use proxmox
::tools
::fs
::{replace_file, CreateOptions}
;
17 use proxmox
::try_block
;
18 use proxmox
::{http_err, identity, list_subdirs_api_method, sortable}
;
20 use pxar
::accessor
::aio
::Accessor
;
23 use crate::api2
::types
::*;
24 use crate::api2
::node
::rrd
::create_value_from_rrd
;
26 use crate::config
::datastore
;
27 use crate::config
::cached_user_info
::CachedUserInfo
;
29 use crate::server
::WorkerTask
;
30 use crate::tools
::{self, AsyncReaderStream, WrappedReaderStream}
;
31 use crate::config
::acl
::{
33 PRIV_DATASTORE_MODIFY
,
36 PRIV_DATASTORE_BACKUP
,
39 fn check_backup_owner(store
: &DataStore
, group
: &BackupGroup
, userid
: &str) -> Result
<(), Error
> {
40 let owner
= store
.get_owner(group
)?
;
42 bail
!("backup owner check failed ({} != {})", userid
, owner
);
47 fn read_backup_index(store
: &DataStore
, backup_dir
: &BackupDir
) -> Result
<Vec
<BackupContent
>, Error
> {
49 let (manifest
, index_size
) = store
.load_manifest(backup_dir
)?
;
51 let mut result
= Vec
::new();
52 for item
in manifest
.files() {
53 result
.push(BackupContent
{
54 filename
: item
.filename
.clone(),
55 encrypted
: item
.encrypted
,
56 size
: Some(item
.size
),
60 result
.push(BackupContent
{
61 filename
: MANIFEST_BLOB_NAME
.to_string(),
62 encrypted
: Some(false),
63 size
: Some(index_size
),
69 fn get_all_snapshot_files(
72 ) -> Result
<Vec
<BackupContent
>, Error
> {
73 let mut files
= read_backup_index(&store
, &info
.backup_dir
)?
;
75 let file_set
= files
.iter().fold(HashSet
::new(), |mut acc
, item
| {
76 acc
.insert(item
.filename
.clone());
80 for file
in &info
.files
{
81 if file_set
.contains(file
) { continue; }
82 files
.push(BackupContent { filename: file.to_string(), size: None, encrypted: None }
);
88 fn group_backups(backup_list
: Vec
<BackupInfo
>) -> HashMap
<String
, Vec
<BackupInfo
>> {
90 let mut group_hash
= HashMap
::new();
92 for info
in backup_list
{
93 let group_id
= info
.backup_dir
.group().group_path().to_str().unwrap().to_owned();
94 let time_list
= group_hash
.entry(group_id
).or_insert(vec
![]);
105 schema
: DATASTORE_SCHEMA
,
111 description
: "Returns the list of backup groups.",
117 permission
: &Permission
::Privilege(
118 &["datastore", "{store}"],
119 PRIV_DATASTORE_AUDIT
| PRIV_DATASTORE_BACKUP
,
123 /// List backup groups.
126 rpcenv
: &mut dyn RpcEnvironment
,
127 ) -> Result
<Vec
<GroupListItem
>, Error
> {
129 let username
= rpcenv
.get_user().unwrap();
130 let user_info
= CachedUserInfo
::new()?
;
131 let user_privs
= user_info
.lookup_privs(&username
, &["datastore", &store
]);
133 let datastore
= DataStore
::lookup_datastore(&store
)?
;
135 let backup_list
= BackupInfo
::list_backups(&datastore
.base_path())?
;
137 let group_hash
= group_backups(backup_list
);
139 let mut groups
= Vec
::new();
141 for (_group_id
, mut list
) in group_hash
{
143 BackupInfo
::sort_list(&mut list
, false);
147 let group
= info
.backup_dir
.group();
149 let list_all
= (user_privs
& PRIV_DATASTORE_AUDIT
) != 0;
150 let owner
= datastore
.get_owner(group
)?
;
152 if owner
!= username { continue; }
155 let result_item
= GroupListItem
{
156 backup_type
: group
.backup_type().to_string(),
157 backup_id
: group
.backup_id().to_string(),
158 last_backup
: info
.backup_dir
.backup_time().timestamp(),
159 backup_count
: list
.len() as u64,
160 files
: info
.files
.clone(),
163 groups
.push(result_item
);
173 schema
: DATASTORE_SCHEMA
,
176 schema
: BACKUP_TYPE_SCHEMA
,
179 schema
: BACKUP_ID_SCHEMA
,
182 schema
: BACKUP_TIME_SCHEMA
,
188 description
: "Returns the list of archive files inside a backup snapshots.",
194 permission
: &Permission
::Privilege(
195 &["datastore", "{store}"],
196 PRIV_DATASTORE_AUDIT
| PRIV_DATASTORE_READ
| PRIV_DATASTORE_BACKUP
,
200 /// List snapshot files.
201 pub fn list_snapshot_files(
207 rpcenv
: &mut dyn RpcEnvironment
,
208 ) -> Result
<Vec
<BackupContent
>, Error
> {
210 let username
= rpcenv
.get_user().unwrap();
211 let user_info
= CachedUserInfo
::new()?
;
212 let user_privs
= user_info
.lookup_privs(&username
, &["datastore", &store
]);
214 let datastore
= DataStore
::lookup_datastore(&store
)?
;
216 let snapshot
= BackupDir
::new(backup_type
, backup_id
, backup_time
);
218 let allowed
= (user_privs
& (PRIV_DATASTORE_AUDIT
| PRIV_DATASTORE_READ
)) != 0;
219 if !allowed { check_backup_owner(&datastore, snapshot.group(), &username)?; }
221 let info
= BackupInfo
::new(&datastore
.base_path(), snapshot
)?
;
223 get_all_snapshot_files(&datastore
, &info
)
230 schema
: DATASTORE_SCHEMA
,
233 schema
: BACKUP_TYPE_SCHEMA
,
236 schema
: BACKUP_ID_SCHEMA
,
239 schema
: BACKUP_TIME_SCHEMA
,
244 permission
: &Permission
::Privilege(
245 &["datastore", "{store}"],
246 PRIV_DATASTORE_MODIFY
| PRIV_DATASTORE_PRUNE
,
250 /// Delete backup snapshot.
257 rpcenv
: &mut dyn RpcEnvironment
,
258 ) -> Result
<Value
, Error
> {
260 let username
= rpcenv
.get_user().unwrap();
261 let user_info
= CachedUserInfo
::new()?
;
262 let user_privs
= user_info
.lookup_privs(&username
, &["datastore", &store
]);
264 let snapshot
= BackupDir
::new(backup_type
, backup_id
, backup_time
);
266 let datastore
= DataStore
::lookup_datastore(&store
)?
;
268 let allowed
= (user_privs
& PRIV_DATASTORE_MODIFY
) != 0;
269 if !allowed { check_backup_owner(&datastore, snapshot.group(), &username)?; }
271 datastore
.remove_backup_dir(&snapshot
)?
;
280 schema
: DATASTORE_SCHEMA
,
284 schema
: BACKUP_TYPE_SCHEMA
,
288 schema
: BACKUP_ID_SCHEMA
,
294 description
: "Returns the list of snapshots.",
296 type: SnapshotListItem
,
300 permission
: &Permission
::Privilege(
301 &["datastore", "{store}"],
302 PRIV_DATASTORE_AUDIT
| PRIV_DATASTORE_BACKUP
,
306 /// List backup snapshots.
307 pub fn list_snapshots (
309 backup_type
: Option
<String
>,
310 backup_id
: Option
<String
>,
313 rpcenv
: &mut dyn RpcEnvironment
,
314 ) -> Result
<Vec
<SnapshotListItem
>, Error
> {
316 let username
= rpcenv
.get_user().unwrap();
317 let user_info
= CachedUserInfo
::new()?
;
318 let user_privs
= user_info
.lookup_privs(&username
, &["datastore", &store
]);
320 let datastore
= DataStore
::lookup_datastore(&store
)?
;
322 let base_path
= datastore
.base_path();
324 let backup_list
= BackupInfo
::list_backups(&base_path
)?
;
326 let mut snapshots
= vec
![];
328 for info
in backup_list
{
329 let group
= info
.backup_dir
.group();
330 if let Some(ref backup_type
) = backup_type
{
331 if backup_type
!= group
.backup_type() { continue; }
333 if let Some(ref backup_id
) = backup_id
{
334 if backup_id
!= group
.backup_id() { continue; }
337 let list_all
= (user_privs
& PRIV_DATASTORE_AUDIT
) != 0;
338 let owner
= datastore
.get_owner(group
)?
;
341 if owner
!= username { continue; }
346 let files
= match get_all_snapshot_files(&datastore
, &info
) {
348 size
= Some(files
.iter().map(|x
| x
.size
.unwrap_or(0)).sum());
352 eprintln
!("error during snapshot file listing: '{}'", err
);
353 info
.files
.iter().map(|x
| BackupContent { filename: x.to_string(), size: None, encrypted: None }
).collect()
357 let result_item
= SnapshotListItem
{
358 backup_type
: group
.backup_type().to_string(),
359 backup_id
: group
.backup_id().to_string(),
360 backup_time
: info
.backup_dir
.backup_time().timestamp(),
366 snapshots
.push(result_item
);
376 schema
: DATASTORE_SCHEMA
,
384 permission
: &Permission
::Privilege(&["datastore", "{store}"], PRIV_DATASTORE_AUDIT
| PRIV_DATASTORE_BACKUP
, true),
387 /// Get datastore status.
391 _rpcenv
: &mut dyn RpcEnvironment
,
392 ) -> Result
<StorageStatus
, Error
> {
393 let datastore
= DataStore
::lookup_datastore(&store
)?
;
394 crate::tools
::disks
::disk_usage(&datastore
.base_path())
401 schema
: DATASTORE_SCHEMA
,
404 schema
: BACKUP_TYPE_SCHEMA
,
408 schema
: BACKUP_ID_SCHEMA
,
412 schema
: BACKUP_TIME_SCHEMA
,
421 permission
: &Permission
::Privilege(&["datastore", "{store}"], PRIV_DATASTORE_READ
| PRIV_DATASTORE_BACKUP
, true), // fixme
426 /// This function can verify a single backup snapshot, all backup from a backup group,
427 /// or all backups in the datastore.
430 backup_type
: Option
<String
>,
431 backup_id
: Option
<String
>,
432 backup_time
: Option
<i64>,
433 rpcenv
: &mut dyn RpcEnvironment
,
434 ) -> Result
<Value
, Error
> {
435 let datastore
= DataStore
::lookup_datastore(&store
)?
;
439 let mut backup_dir
= None
;
440 let mut backup_group
= None
;
442 match (backup_type
, backup_id
, backup_time
) {
443 (Some(backup_type
), Some(backup_id
), Some(backup_time
)) => {
444 worker_id
= format
!("{}_{}_{}_{:08X}", store
, backup_type
, backup_id
, backup_time
);
445 let dir
= BackupDir
::new(backup_type
, backup_id
, backup_time
);
446 backup_dir
= Some(dir
);
448 (Some(backup_type
), Some(backup_id
), None
) => {
449 worker_id
= format
!("{}_{}_{}", store
, backup_type
, backup_id
);
450 let group
= BackupGroup
::new(backup_type
, backup_id
);
451 backup_group
= Some(group
);
453 (None
, None
, None
) => {
454 worker_id
= store
.clone();
456 _
=> bail
!("parameters do not spefify a backup group or snapshot"),
459 let username
= rpcenv
.get_user().unwrap();
460 let to_stdout
= if rpcenv
.env_type() == RpcEnvironmentType
::CLI { true }
else { false }
;
462 let upid_str
= WorkerTask
::new_thread(
463 "verify", Some(worker_id
.clone()), &username
, to_stdout
, move |worker
|
465 let success
= if let Some(backup_dir
) = backup_dir
{
466 verify_backup_dir(&datastore
, &backup_dir
, &worker
)?
467 } else if let Some(backup_group
) = backup_group
{
468 verify_backup_group(&datastore
, &backup_group
, &worker
)?
470 verify_all_backups(&datastore
, &worker
)?
473 bail
!("verfication failed - please check the log for details");
482 macro_rules
! add_common_prune_prameters
{
483 ( [ $
( $list1
:tt
)* ] ) => {
484 add_common_prune_prameters
!([$
( $list1
)* ] , [])
486 ( [ $
( $list1
:tt
)* ] , [ $
( $list2
:tt
)* ] ) => {
492 &PRUNE_SCHEMA_KEEP_DAILY
,
497 &PRUNE_SCHEMA_KEEP_HOURLY
,
502 &PRUNE_SCHEMA_KEEP_LAST
,
507 &PRUNE_SCHEMA_KEEP_MONTHLY
,
512 &PRUNE_SCHEMA_KEEP_WEEKLY
,
517 &PRUNE_SCHEMA_KEEP_YEARLY
,
524 pub const API_RETURN_SCHEMA_PRUNE
: Schema
= ArraySchema
::new(
525 "Returns the list of snapshots and a flag indicating if there are kept or removed.",
526 PruneListItem
::API_SCHEMA
529 const API_METHOD_PRUNE
: ApiMethod
= ApiMethod
::new(
530 &ApiHandler
::Sync(&prune
),
532 "Prune the datastore.",
533 &add_common_prune_prameters
!([
534 ("backup-id", false, &BACKUP_ID_SCHEMA
),
535 ("backup-type", false, &BACKUP_TYPE_SCHEMA
),
536 ("dry-run", true, &BooleanSchema
::new(
537 "Just show what prune would do, but do not delete anything.")
541 ("store", false, &DATASTORE_SCHEMA
),
544 .returns(&API_RETURN_SCHEMA_PRUNE
)
545 .access(None
, &Permission
::Privilege(
546 &["datastore", "{store}"],
547 PRIV_DATASTORE_MODIFY
| PRIV_DATASTORE_PRUNE
,
554 rpcenv
: &mut dyn RpcEnvironment
,
555 ) -> Result
<Value
, Error
> {
557 let store
= tools
::required_string_param(¶m
, "store")?
;
558 let backup_type
= tools
::required_string_param(¶m
, "backup-type")?
;
559 let backup_id
= tools
::required_string_param(¶m
, "backup-id")?
;
561 let username
= rpcenv
.get_user().unwrap();
562 let user_info
= CachedUserInfo
::new()?
;
563 let user_privs
= user_info
.lookup_privs(&username
, &["datastore", &store
]);
565 let dry_run
= param
["dry-run"].as_bool().unwrap_or(false);
567 let group
= BackupGroup
::new(backup_type
, backup_id
);
569 let datastore
= DataStore
::lookup_datastore(&store
)?
;
571 let allowed
= (user_privs
& PRIV_DATASTORE_MODIFY
) != 0;
572 if !allowed { check_backup_owner(&datastore, &group, &username)?; }
574 let prune_options
= PruneOptions
{
575 keep_last
: param
["keep-last"].as_u64(),
576 keep_hourly
: param
["keep-hourly"].as_u64(),
577 keep_daily
: param
["keep-daily"].as_u64(),
578 keep_weekly
: param
["keep-weekly"].as_u64(),
579 keep_monthly
: param
["keep-monthly"].as_u64(),
580 keep_yearly
: param
["keep-yearly"].as_u64(),
583 let worker_id
= format
!("{}_{}_{}", store
, backup_type
, backup_id
);
585 let mut prune_result
= Vec
::new();
587 let list
= group
.list_backups(&datastore
.base_path())?
;
589 let mut prune_info
= compute_prune_info(list
, &prune_options
)?
;
591 prune_info
.reverse(); // delete older snapshots first
593 let keep_all
= !prune_options
.keeps_something();
596 for (info
, mut keep
) in prune_info
{
597 if keep_all { keep = true; }
599 let backup_time
= info
.backup_dir
.backup_time();
600 let group
= info
.backup_dir
.group();
602 prune_result
.push(json
!({
603 "backup-type": group
.backup_type(),
604 "backup-id": group
.backup_id(),
605 "backup-time": backup_time
.timestamp(),
609 return Ok(json
!(prune_result
));
613 // We use a WorkerTask just to have a task log, but run synchrounously
614 let worker
= WorkerTask
::new("prune", Some(worker_id
), "root@pam", true)?
;
616 let result
= try_block
! {
618 worker
.log("No prune selection - keeping all files.");
620 worker
.log(format
!("retention options: {}", prune_options
.cli_options_string()));
621 worker
.log(format
!("Starting prune on store \"{}\" group \"{}/{}\"",
622 store
, backup_type
, backup_id
));
625 for (info
, mut keep
) in prune_info
{
626 if keep_all { keep = true; }
628 let backup_time
= info
.backup_dir
.backup_time();
629 let timestamp
= BackupDir
::backup_time_to_string(backup_time
);
630 let group
= info
.backup_dir
.group();
638 if keep { "keep" }
else { "remove" }
,
643 prune_result
.push(json
!({
644 "backup-type": group
.backup_type(),
645 "backup-id": group
.backup_id(),
646 "backup-time": backup_time
.timestamp(),
650 if !(dry_run
|| keep
) {
651 datastore
.remove_backup_dir(&info
.backup_dir
)?
;
658 worker
.log_result(&result
);
660 if let Err(err
) = result
{
661 bail
!("prune failed - {}", err
);
664 Ok(json
!(prune_result
))
671 schema
: DATASTORE_SCHEMA
,
679 permission
: &Permission
::Privilege(&["datastore", "{store}"], PRIV_DATASTORE_MODIFY
, false),
682 /// Start garbage collection.
683 fn start_garbage_collection(
686 rpcenv
: &mut dyn RpcEnvironment
,
687 ) -> Result
<Value
, Error
> {
689 let datastore
= DataStore
::lookup_datastore(&store
)?
;
691 println
!("Starting garbage collection on store {}", store
);
693 let to_stdout
= if rpcenv
.env_type() == RpcEnvironmentType
::CLI { true }
else { false }
;
695 let upid_str
= WorkerTask
::new_thread(
696 "garbage_collection", Some(store
.clone()), "root@pam", to_stdout
, move |worker
|
698 worker
.log(format
!("starting garbage collection on store {}", store
));
699 datastore
.garbage_collection(&worker
)
709 schema
: DATASTORE_SCHEMA
,
714 type: GarbageCollectionStatus
,
717 permission
: &Permission
::Privilege(&["datastore", "{store}"], PRIV_DATASTORE_AUDIT
, false),
720 /// Garbage collection status.
721 pub fn garbage_collection_status(
724 _rpcenv
: &mut dyn RpcEnvironment
,
725 ) -> Result
<GarbageCollectionStatus
, Error
> {
727 let datastore
= DataStore
::lookup_datastore(&store
)?
;
729 let status
= datastore
.last_gc_status();
736 description
: "List the accessible datastores.",
739 description
: "Datastore name and description.",
742 schema
: DATASTORE_SCHEMA
,
746 schema
: SINGLE_LINE_COMMENT_SCHEMA
,
752 permission
: &Permission
::Anybody
,
756 fn get_datastore_list(
759 rpcenv
: &mut dyn RpcEnvironment
,
760 ) -> Result
<Value
, Error
> {
762 let (config
, _digest
) = datastore
::config()?
;
764 let username
= rpcenv
.get_user().unwrap();
765 let user_info
= CachedUserInfo
::new()?
;
767 let mut list
= Vec
::new();
769 for (store
, (_
, data
)) in &config
.sections
{
770 let user_privs
= user_info
.lookup_privs(&username
, &["datastore", &store
]);
771 let allowed
= (user_privs
& (PRIV_DATASTORE_AUDIT
| PRIV_DATASTORE_BACKUP
)) != 0;
773 let mut entry
= json
!({ "store": store }
);
774 if let Some(comment
) = data
["comment"].as_str() {
775 entry
["comment"] = comment
.into();
785 pub const API_METHOD_DOWNLOAD_FILE
: ApiMethod
= ApiMethod
::new(
786 &ApiHandler
::AsyncHttp(&download_file
),
788 "Download single raw file from backup snapshot.",
790 ("store", false, &DATASTORE_SCHEMA
),
791 ("backup-type", false, &BACKUP_TYPE_SCHEMA
),
792 ("backup-id", false, &BACKUP_ID_SCHEMA
),
793 ("backup-time", false, &BACKUP_TIME_SCHEMA
),
794 ("file-name", false, &BACKUP_ARCHIVE_NAME_SCHEMA
),
797 ).access(None
, &Permission
::Privilege(
798 &["datastore", "{store}"],
799 PRIV_DATASTORE_READ
| PRIV_DATASTORE_BACKUP
,
808 rpcenv
: Box
<dyn RpcEnvironment
>,
809 ) -> ApiResponseFuture
{
812 let store
= tools
::required_string_param(¶m
, "store")?
;
813 let datastore
= DataStore
::lookup_datastore(store
)?
;
815 let username
= rpcenv
.get_user().unwrap();
816 let user_info
= CachedUserInfo
::new()?
;
817 let user_privs
= user_info
.lookup_privs(&username
, &["datastore", &store
]);
819 let file_name
= tools
::required_string_param(¶m
, "file-name")?
.to_owned();
821 let backup_type
= tools
::required_string_param(¶m
, "backup-type")?
;
822 let backup_id
= tools
::required_string_param(¶m
, "backup-id")?
;
823 let backup_time
= tools
::required_integer_param(¶m
, "backup-time")?
;
825 let backup_dir
= BackupDir
::new(backup_type
, backup_id
, backup_time
);
827 let allowed
= (user_privs
& PRIV_DATASTORE_READ
) != 0;
828 if !allowed { check_backup_owner(&datastore, backup_dir.group(), &username)?; }
830 println
!("Download {} from {} ({}/{})", file_name
, store
, backup_dir
, file_name
);
832 let mut path
= datastore
.base_path();
833 path
.push(backup_dir
.relative_path());
834 path
.push(&file_name
);
836 let file
= tokio
::fs
::File
::open(&path
)
837 .map_err(|err
| http_err
!(BAD_REQUEST
, format
!("File open failed: {}", err
)))
840 let payload
= tokio_util
::codec
::FramedRead
::new(file
, tokio_util
::codec
::BytesCodec
::new())
841 .map_ok(|bytes
| hyper
::body
::Bytes
::from(bytes
.freeze()))
842 .map_err(move |err
| {
843 eprintln
!("error during streaming of '{:?}' - {}", &path
, err
);
846 let body
= Body
::wrap_stream(payload
);
848 // fixme: set other headers ?
849 Ok(Response
::builder()
850 .status(StatusCode
::OK
)
851 .header(header
::CONTENT_TYPE
, "application/octet-stream")
858 pub const API_METHOD_DOWNLOAD_FILE_DECODED
: ApiMethod
= ApiMethod
::new(
859 &ApiHandler
::AsyncHttp(&download_file_decoded
),
861 "Download single decoded file from backup snapshot. Only works if it's not encrypted.",
863 ("store", false, &DATASTORE_SCHEMA
),
864 ("backup-type", false, &BACKUP_TYPE_SCHEMA
),
865 ("backup-id", false, &BACKUP_ID_SCHEMA
),
866 ("backup-time", false, &BACKUP_TIME_SCHEMA
),
867 ("file-name", false, &BACKUP_ARCHIVE_NAME_SCHEMA
),
870 ).access(None
, &Permission
::Privilege(
871 &["datastore", "{store}"],
872 PRIV_DATASTORE_READ
| PRIV_DATASTORE_BACKUP
,
876 fn download_file_decoded(
881 rpcenv
: Box
<dyn RpcEnvironment
>,
882 ) -> ApiResponseFuture
{
885 let store
= tools
::required_string_param(¶m
, "store")?
;
886 let datastore
= DataStore
::lookup_datastore(store
)?
;
888 let username
= rpcenv
.get_user().unwrap();
889 let user_info
= CachedUserInfo
::new()?
;
890 let user_privs
= user_info
.lookup_privs(&username
, &["datastore", &store
]);
892 let file_name
= tools
::required_string_param(¶m
, "file-name")?
.to_owned();
894 let backup_type
= tools
::required_string_param(¶m
, "backup-type")?
;
895 let backup_id
= tools
::required_string_param(¶m
, "backup-id")?
;
896 let backup_time
= tools
::required_integer_param(¶m
, "backup-time")?
;
898 let backup_dir
= BackupDir
::new(backup_type
, backup_id
, backup_time
);
900 let allowed
= (user_privs
& PRIV_DATASTORE_READ
) != 0;
901 if !allowed { check_backup_owner(&datastore, backup_dir.group(), &username)?; }
903 let files
= read_backup_index(&datastore
, &backup_dir
)?
;
905 if file
.filename
== file_name
&& file
.encrypted
== Some(true) {
906 bail
!("cannot decode '{}' - is encrypted", file_name
);
910 println
!("Download {} from {} ({}/{})", file_name
, store
, backup_dir
, file_name
);
912 let mut path
= datastore
.base_path();
913 path
.push(backup_dir
.relative_path());
914 path
.push(&file_name
);
916 let extension
= file_name
.rsplitn(2, '
.'
).next().unwrap();
918 let body
= match extension
{
920 let index
= DynamicIndexReader
::open(&path
)
921 .map_err(|err
| format_err
!("unable to read dynamic index '{:?}' - {}", &path
, err
))?
;
923 let chunk_reader
= LocalChunkReader
::new(datastore
, None
);
924 let reader
= AsyncIndexReader
::new(index
, chunk_reader
);
925 Body
::wrap_stream(AsyncReaderStream
::new(reader
)
926 .map_err(move |err
| {
927 eprintln
!("error during streaming of '{:?}' - {}", path
, err
);
932 let index
= FixedIndexReader
::open(&path
)
933 .map_err(|err
| format_err
!("unable to read fixed index '{:?}' - {}", &path
, err
))?
;
935 let chunk_reader
= LocalChunkReader
::new(datastore
, None
);
936 let reader
= AsyncIndexReader
::new(index
, chunk_reader
);
937 Body
::wrap_stream(AsyncReaderStream
::with_buffer_size(reader
, 4*1024*1024)
938 .map_err(move |err
| {
939 eprintln
!("error during streaming of '{:?}' - {}", path
, err
);
944 let file
= std
::fs
::File
::open(&path
)
945 .map_err(|err
| http_err
!(BAD_REQUEST
, format
!("File open failed: {}", err
)))?
;
948 WrappedReaderStream
::new(DataBlobReader
::new(file
, None
)?
)
949 .map_err(move |err
| {
950 eprintln
!("error during streaming of '{:?}' - {}", path
, err
);
956 bail
!("cannot download '{}' files", extension
);
960 // fixme: set other headers ?
961 Ok(Response
::builder()
962 .status(StatusCode
::OK
)
963 .header(header
::CONTENT_TYPE
, "application/octet-stream")
970 pub const API_METHOD_UPLOAD_BACKUP_LOG
: ApiMethod
= ApiMethod
::new(
971 &ApiHandler
::AsyncHttp(&upload_backup_log
),
973 "Upload the client backup log file into a backup snapshot ('client.log.blob').",
975 ("store", false, &DATASTORE_SCHEMA
),
976 ("backup-type", false, &BACKUP_TYPE_SCHEMA
),
977 ("backup-id", false, &BACKUP_ID_SCHEMA
),
978 ("backup-time", false, &BACKUP_TIME_SCHEMA
),
982 Some("Only the backup creator/owner is allowed to do this."),
983 &Permission
::Privilege(&["datastore", "{store}"], PRIV_DATASTORE_BACKUP
, false)
986 fn upload_backup_log(
991 rpcenv
: Box
<dyn RpcEnvironment
>,
992 ) -> ApiResponseFuture
{
995 let store
= tools
::required_string_param(¶m
, "store")?
;
996 let datastore
= DataStore
::lookup_datastore(store
)?
;
998 let file_name
= CLIENT_LOG_BLOB_NAME
;
1000 let backup_type
= tools
::required_string_param(¶m
, "backup-type")?
;
1001 let backup_id
= tools
::required_string_param(¶m
, "backup-id")?
;
1002 let backup_time
= tools
::required_integer_param(¶m
, "backup-time")?
;
1004 let backup_dir
= BackupDir
::new(backup_type
, backup_id
, backup_time
);
1006 let username
= rpcenv
.get_user().unwrap();
1007 check_backup_owner(&datastore
, backup_dir
.group(), &username
)?
;
1009 let mut path
= datastore
.base_path();
1010 path
.push(backup_dir
.relative_path());
1011 path
.push(&file_name
);
1014 bail
!("backup already contains a log.");
1017 println
!("Upload backup log to {}/{}/{}/{}/{}", store
,
1018 backup_type
, backup_id
, BackupDir
::backup_time_to_string(backup_dir
.backup_time()), file_name
);
1021 .map_err(Error
::from
)
1022 .try_fold(Vec
::new(), |mut acc
, chunk
| {
1023 acc
.extend_from_slice(&*chunk
);
1024 future
::ok
::<_
, Error
>(acc
)
1028 let blob
= DataBlob
::from_raw(data
)?
;
1029 // always verify CRC at server side
1031 let raw_data
= blob
.raw_data();
1032 replace_file(&path
, raw_data
, CreateOptions
::new())?
;
1034 // fixme: use correct formatter
1035 Ok(crate::server
::formatter
::json_response(Ok(Value
::Null
)))
1043 schema
: DATASTORE_SCHEMA
,
1046 schema
: BACKUP_TYPE_SCHEMA
,
1049 schema
: BACKUP_ID_SCHEMA
,
1052 schema
: BACKUP_TIME_SCHEMA
,
1055 description
: "Base64 encoded path.",
1061 permission
: &Permission
::Privilege(&["datastore", "{store}"], PRIV_DATASTORE_READ
| PRIV_DATASTORE_BACKUP
, true),
1064 /// Get the entries of the given path of the catalog
1067 backup_type
: String
,
1073 rpcenv
: &mut dyn RpcEnvironment
,
1074 ) -> Result
<Value
, Error
> {
1075 let datastore
= DataStore
::lookup_datastore(&store
)?
;
1077 let username
= rpcenv
.get_user().unwrap();
1078 let user_info
= CachedUserInfo
::new()?
;
1079 let user_privs
= user_info
.lookup_privs(&username
, &["datastore", &store
]);
1081 let backup_dir
= BackupDir
::new(backup_type
, backup_id
, backup_time
);
1083 let allowed
= (user_privs
& PRIV_DATASTORE_READ
) != 0;
1084 if !allowed { check_backup_owner(&datastore, backup_dir.group(), &username)?; }
1086 let mut path
= datastore
.base_path();
1087 path
.push(backup_dir
.relative_path());
1088 path
.push(CATALOG_NAME
);
1090 let index
= DynamicIndexReader
::open(&path
)
1091 .map_err(|err
| format_err
!("unable to read dynamic index '{:?}' - {}", &path
, err
))?
;
1093 let chunk_reader
= LocalChunkReader
::new(datastore
, None
);
1094 let reader
= BufferedDynamicReader
::new(index
, chunk_reader
);
1096 let mut catalog_reader
= CatalogReader
::new(reader
);
1097 let mut current
= catalog_reader
.root()?
;
1098 let mut components
= vec
![];
1101 if filepath
!= "root" {
1102 components
= base64
::decode(filepath
)?
;
1103 if components
.len() > 0 && components
[0] == '
/'
as u8 {
1104 components
.remove(0);
1106 for component
in components
.split(|c
| *c
== '
/'
as u8) {
1107 if let Some(entry
) = catalog_reader
.lookup(¤t
, component
)?
{
1110 bail
!("path {:?} not found in catalog", &String
::from_utf8_lossy(&components
));
1115 let mut res
= Vec
::new();
1117 for direntry
in catalog_reader
.read_dir(¤t
)?
{
1118 let mut components
= components
.clone();
1119 components
.push('
/'
as u8);
1120 components
.extend(&direntry
.name
);
1121 let path
= base64
::encode(components
);
1122 let text
= String
::from_utf8_lossy(&direntry
.name
);
1123 let mut entry
= json
!({
1126 "type": CatalogEntryType
::from(&direntry
.attr
).to_string(),
1129 match direntry
.attr
{
1130 DirEntryAttribute
::Directory { start: _ }
=> {
1131 entry
["leaf"] = false.into();
1133 DirEntryAttribute
::File { size, mtime }
=> {
1134 entry
["size"] = size
.into();
1135 entry
["mtime"] = mtime
.into();
1146 pub const API_METHOD_PXAR_FILE_DOWNLOAD
: ApiMethod
= ApiMethod
::new(
1147 &ApiHandler
::AsyncHttp(&pxar_file_download
),
1149 "Download single file from pxar file of a bacup snapshot. Only works if it's not encrypted.",
1151 ("store", false, &DATASTORE_SCHEMA
),
1152 ("backup-type", false, &BACKUP_TYPE_SCHEMA
),
1153 ("backup-id", false, &BACKUP_ID_SCHEMA
),
1154 ("backup-time", false, &BACKUP_TIME_SCHEMA
),
1155 ("filepath", false, &StringSchema
::new("Base64 encoded path").schema()),
1158 ).access(None
, &Permission
::Privilege(
1159 &["datastore", "{store}"],
1160 PRIV_DATASTORE_READ
| PRIV_DATASTORE_BACKUP
,
1164 fn pxar_file_download(
1169 rpcenv
: Box
<dyn RpcEnvironment
>,
1170 ) -> ApiResponseFuture
{
1173 let store
= tools
::required_string_param(¶m
, "store")?
;
1174 let datastore
= DataStore
::lookup_datastore(&store
)?
;
1176 let username
= rpcenv
.get_user().unwrap();
1177 let user_info
= CachedUserInfo
::new()?
;
1178 let user_privs
= user_info
.lookup_privs(&username
, &["datastore", &store
]);
1180 let filepath
= tools
::required_string_param(¶m
, "filepath")?
.to_owned();
1182 let backup_type
= tools
::required_string_param(¶m
, "backup-type")?
;
1183 let backup_id
= tools
::required_string_param(¶m
, "backup-id")?
;
1184 let backup_time
= tools
::required_integer_param(¶m
, "backup-time")?
;
1186 let backup_dir
= BackupDir
::new(backup_type
, backup_id
, backup_time
);
1188 let allowed
= (user_privs
& PRIV_DATASTORE_READ
) != 0;
1189 if !allowed { check_backup_owner(&datastore, backup_dir.group(), &username)?; }
1191 let mut path
= datastore
.base_path();
1192 path
.push(backup_dir
.relative_path());
1194 let mut components
= base64
::decode(&filepath
)?
;
1195 if components
.len() > 0 && components
[0] == '
/'
as u8 {
1196 components
.remove(0);
1199 let mut split
= components
.splitn(2, |c
| *c
== '
/'
as u8);
1200 let pxar_name
= split
.next().unwrap();
1201 let file_path
= split
.next().ok_or(format_err
!("filepath looks strange '{}'", filepath
))?
;
1203 path
.push(OsStr
::from_bytes(&pxar_name
));
1205 let index
= DynamicIndexReader
::open(&path
)
1206 .map_err(|err
| format_err
!("unable to read dynamic index '{:?}' - {}", &path
, err
))?
;
1208 let chunk_reader
= LocalChunkReader
::new(datastore
, None
);
1209 let reader
= BufferedDynamicReader
::new(index
, chunk_reader
);
1210 let archive_size
= reader
.archive_size();
1211 let reader
= LocalDynamicReadAt
::new(reader
);
1213 let decoder
= Accessor
::new(reader
, archive_size
).await?
;
1214 let root
= decoder
.open_root().await?
;
1216 .lookup(OsStr
::from_bytes(file_path
)).await?
1217 .ok_or(format_err
!("error opening '{:?}'", file_path
))?
;
1219 let file
= match file
.kind() {
1220 EntryKind
::File { .. }
=> file
,
1221 EntryKind
::Hardlink(_
) => {
1222 decoder
.follow_hardlink(&file
).await?
1225 other
=> bail
!("cannot download file of type {:?}", other
),
1228 let body
= Body
::wrap_stream(
1229 AsyncReaderStream
::new(file
.contents().await?
)
1230 .map_err(move |err
| {
1231 eprintln
!("error during streaming of '{:?}' - {}", filepath
, err
);
1236 // fixme: set other headers ?
1237 Ok(Response
::builder()
1238 .status(StatusCode
::OK
)
1239 .header(header
::CONTENT_TYPE
, "application/octet-stream")
1249 schema
: DATASTORE_SCHEMA
,
1252 type: RRDTimeFrameResolution
,
1260 permission
: &Permission
::Privilege(&["datastore", "{store}"], PRIV_DATASTORE_AUDIT
| PRIV_DATASTORE_BACKUP
, true),
1263 /// Read datastore stats
1266 timeframe
: RRDTimeFrameResolution
,
1269 ) -> Result
<Value
, Error
> {
1271 create_value_from_rrd(
1272 &format
!("datastore/{}", store
),
1275 "read_ios", "read_bytes",
1276 "write_ios", "write_bytes",
1285 const DATASTORE_INFO_SUBDIRS
: SubdirMap
= &[
1289 .get(&API_METHOD_CATALOG
)
1294 .download(&API_METHOD_DOWNLOAD_FILE
)
1299 .download(&API_METHOD_DOWNLOAD_FILE_DECODED
)
1304 .get(&API_METHOD_LIST_SNAPSHOT_FILES
)
1309 .get(&API_METHOD_GARBAGE_COLLECTION_STATUS
)
1310 .post(&API_METHOD_START_GARBAGE_COLLECTION
)
1315 .get(&API_METHOD_LIST_GROUPS
)
1320 .post(&API_METHOD_PRUNE
)
1323 "pxar-file-download",
1325 .download(&API_METHOD_PXAR_FILE_DOWNLOAD
)
1330 .get(&API_METHOD_GET_RRD_STATS
)
1335 .get(&API_METHOD_LIST_SNAPSHOTS
)
1336 .delete(&API_METHOD_DELETE_SNAPSHOT
)
1341 .get(&API_METHOD_STATUS
)
1344 "upload-backup-log",
1346 .upload(&API_METHOD_UPLOAD_BACKUP_LOG
)
1351 .post(&API_METHOD_VERIFY
)
1355 const DATASTORE_INFO_ROUTER
: Router
= Router
::new()
1356 .get(&list_subdirs_api_method
!(DATASTORE_INFO_SUBDIRS
))
1357 .subdirs(DATASTORE_INFO_SUBDIRS
);
1360 pub const ROUTER
: Router
= Router
::new()
1361 .get(&API_METHOD_GET_DATASTORE_LIST
)
1362 .match_all("store", &DATASTORE_INFO_ROUTER
);