1 use std
::collections
::{HashSet, HashMap}
;
3 use std
::os
::unix
::ffi
::OsStrExt
;
4 use std
::sync
::{Arc, Mutex}
;
5 use std
::path
::{Path, PathBuf}
;
8 use anyhow
::{bail, format_err, Error}
;
10 use hyper
::http
::request
::Parts
;
11 use hyper
::{header, Body, Response, StatusCode}
;
12 use serde_json
::{json, Value}
;
15 api
, ApiResponseFuture
, ApiHandler
, ApiMethod
, Router
,
16 RpcEnvironment
, RpcEnvironmentType
, Permission
18 use proxmox
::api
::router
::SubdirMap
;
19 use proxmox
::api
::schema
::*;
20 use proxmox
::tools
::fs
::{replace_file, CreateOptions}
;
21 use proxmox
::{http_err, identity, list_subdirs_api_method, sortable}
;
23 use pxar
::accessor
::aio
::{Accessor, FileContents, FileEntry}
;
26 use crate::api2
::types
::*;
27 use crate::api2
::node
::rrd
::create_value_from_rrd
;
29 use crate::config
::datastore
;
30 use crate::config
::cached_user_info
::CachedUserInfo
;
32 use crate::server
::WorkerTask
;
35 zip
::{ZipEncoder, ZipEntry}
,
36 AsyncChannelWriter
, AsyncReaderStream
, WrappedReaderStream
,
39 use crate::config
::acl
::{
41 PRIV_DATASTORE_MODIFY
,
44 PRIV_DATASTORE_BACKUP
,
47 fn check_backup_owner(
51 ) -> Result
<(), Error
> {
52 let owner
= store
.get_owner(group
)?
;
54 bail
!("backup owner check failed ({} != {})", userid
, owner
);
61 backup_dir
: &BackupDir
,
62 ) -> Result
<(BackupManifest
, Vec
<BackupContent
>), Error
> {
64 let (manifest
, index_size
) = store
.load_manifest(backup_dir
)?
;
66 let mut result
= Vec
::new();
67 for item
in manifest
.files() {
68 result
.push(BackupContent
{
69 filename
: item
.filename
.clone(),
70 crypt_mode
: Some(item
.crypt_mode
),
71 size
: Some(item
.size
),
75 result
.push(BackupContent
{
76 filename
: MANIFEST_BLOB_NAME
.to_string(),
77 crypt_mode
: match manifest
.signature
{
78 Some(_
) => Some(CryptMode
::SignOnly
),
79 None
=> Some(CryptMode
::None
),
81 size
: Some(index_size
),
84 Ok((manifest
, result
))
87 fn get_all_snapshot_files(
90 ) -> Result
<(BackupManifest
, Vec
<BackupContent
>), Error
> {
92 let (manifest
, mut files
) = read_backup_index(&store
, &info
.backup_dir
)?
;
94 let file_set
= files
.iter().fold(HashSet
::new(), |mut acc
, item
| {
95 acc
.insert(item
.filename
.clone());
99 for file
in &info
.files
{
100 if file_set
.contains(file
) { continue; }
101 files
.push(BackupContent
{
102 filename
: file
.to_string(),
108 Ok((manifest
, files
))
111 fn group_backups(backup_list
: Vec
<BackupInfo
>) -> HashMap
<String
, Vec
<BackupInfo
>> {
113 let mut group_hash
= HashMap
::new();
115 for info
in backup_list
{
116 let group_id
= info
.backup_dir
.group().group_path().to_str().unwrap().to_owned();
117 let time_list
= group_hash
.entry(group_id
).or_insert(vec
![]);
118 time_list
.push(info
);
128 schema
: DATASTORE_SCHEMA
,
134 description
: "Returns the list of backup groups.",
140 permission
: &Permission
::Privilege(
141 &["datastore", "{store}"],
142 PRIV_DATASTORE_AUDIT
| PRIV_DATASTORE_BACKUP
,
146 /// List backup groups.
149 rpcenv
: &mut dyn RpcEnvironment
,
150 ) -> Result
<Vec
<GroupListItem
>, Error
> {
152 let userid
: Userid
= rpcenv
.get_user().unwrap().parse()?
;
153 let user_info
= CachedUserInfo
::new()?
;
154 let user_privs
= user_info
.lookup_privs(&userid
, &["datastore", &store
]);
156 let datastore
= DataStore
::lookup_datastore(&store
)?
;
158 let backup_list
= BackupInfo
::list_backups(&datastore
.base_path())?
;
160 let group_hash
= group_backups(backup_list
);
162 let mut groups
= Vec
::new();
164 for (_group_id
, mut list
) in group_hash
{
166 BackupInfo
::sort_list(&mut list
, false);
170 let group
= info
.backup_dir
.group();
172 let list_all
= (user_privs
& PRIV_DATASTORE_AUDIT
) != 0;
173 let owner
= datastore
.get_owner(group
)?
;
174 if !list_all
&& owner
!= userid
{
178 let result_item
= GroupListItem
{
179 backup_type
: group
.backup_type().to_string(),
180 backup_id
: group
.backup_id().to_string(),
181 last_backup
: info
.backup_dir
.backup_time(),
182 backup_count
: list
.len() as u64,
183 files
: info
.files
.clone(),
186 groups
.push(result_item
);
196 schema
: DATASTORE_SCHEMA
,
199 schema
: BACKUP_TYPE_SCHEMA
,
202 schema
: BACKUP_ID_SCHEMA
,
205 schema
: BACKUP_TIME_SCHEMA
,
211 description
: "Returns the list of archive files inside a backup snapshots.",
217 permission
: &Permission
::Privilege(
218 &["datastore", "{store}"],
219 PRIV_DATASTORE_AUDIT
| PRIV_DATASTORE_READ
| PRIV_DATASTORE_BACKUP
,
223 /// List snapshot files.
224 pub fn list_snapshot_files(
230 rpcenv
: &mut dyn RpcEnvironment
,
231 ) -> Result
<Vec
<BackupContent
>, Error
> {
233 let userid
: Userid
= rpcenv
.get_user().unwrap().parse()?
;
234 let user_info
= CachedUserInfo
::new()?
;
235 let user_privs
= user_info
.lookup_privs(&userid
, &["datastore", &store
]);
237 let datastore
= DataStore
::lookup_datastore(&store
)?
;
239 let snapshot
= BackupDir
::new(backup_type
, backup_id
, backup_time
)?
;
241 let allowed
= (user_privs
& (PRIV_DATASTORE_AUDIT
| PRIV_DATASTORE_READ
)) != 0;
242 if !allowed { check_backup_owner(&datastore, snapshot.group(), &userid)?; }
244 let info
= BackupInfo
::new(&datastore
.base_path(), snapshot
)?
;
246 let (_manifest
, files
) = get_all_snapshot_files(&datastore
, &info
)?
;
255 schema
: DATASTORE_SCHEMA
,
258 schema
: BACKUP_TYPE_SCHEMA
,
261 schema
: BACKUP_ID_SCHEMA
,
264 schema
: BACKUP_TIME_SCHEMA
,
269 permission
: &Permission
::Privilege(
270 &["datastore", "{store}"],
271 PRIV_DATASTORE_MODIFY
| PRIV_DATASTORE_PRUNE
,
275 /// Delete backup snapshot.
282 rpcenv
: &mut dyn RpcEnvironment
,
283 ) -> Result
<Value
, Error
> {
285 let userid
: Userid
= rpcenv
.get_user().unwrap().parse()?
;
286 let user_info
= CachedUserInfo
::new()?
;
287 let user_privs
= user_info
.lookup_privs(&userid
, &["datastore", &store
]);
289 let snapshot
= BackupDir
::new(backup_type
, backup_id
, backup_time
)?
;
291 let datastore
= DataStore
::lookup_datastore(&store
)?
;
293 let allowed
= (user_privs
& PRIV_DATASTORE_MODIFY
) != 0;
294 if !allowed { check_backup_owner(&datastore, snapshot.group(), &userid)?; }
296 datastore
.remove_backup_dir(&snapshot
, false)?
;
305 schema
: DATASTORE_SCHEMA
,
309 schema
: BACKUP_TYPE_SCHEMA
,
313 schema
: BACKUP_ID_SCHEMA
,
319 description
: "Returns the list of snapshots.",
321 type: SnapshotListItem
,
325 permission
: &Permission
::Privilege(
326 &["datastore", "{store}"],
327 PRIV_DATASTORE_AUDIT
| PRIV_DATASTORE_BACKUP
,
331 /// List backup snapshots.
332 pub fn list_snapshots (
334 backup_type
: Option
<String
>,
335 backup_id
: Option
<String
>,
338 rpcenv
: &mut dyn RpcEnvironment
,
339 ) -> Result
<Vec
<SnapshotListItem
>, Error
> {
341 let userid
: Userid
= rpcenv
.get_user().unwrap().parse()?
;
342 let user_info
= CachedUserInfo
::new()?
;
343 let user_privs
= user_info
.lookup_privs(&userid
, &["datastore", &store
]);
345 let datastore
= DataStore
::lookup_datastore(&store
)?
;
347 let base_path
= datastore
.base_path();
349 let backup_list
= BackupInfo
::list_backups(&base_path
)?
;
351 let mut snapshots
= vec
![];
353 for info
in backup_list
{
354 let group
= info
.backup_dir
.group();
355 if let Some(ref backup_type
) = backup_type
{
356 if backup_type
!= group
.backup_type() { continue; }
358 if let Some(ref backup_id
) = backup_id
{
359 if backup_id
!= group
.backup_id() { continue; }
362 let list_all
= (user_privs
& PRIV_DATASTORE_AUDIT
) != 0;
363 let owner
= datastore
.get_owner(group
)?
;
365 if !list_all
&& owner
!= userid
{
371 let (comment
, verification
, files
) = match get_all_snapshot_files(&datastore
, &info
) {
372 Ok((manifest
, files
)) => {
373 size
= Some(files
.iter().map(|x
| x
.size
.unwrap_or(0)).sum());
374 // extract the first line from notes
375 let comment
: Option
<String
> = manifest
.unprotected
["notes"]
377 .and_then(|notes
| notes
.lines().next())
380 let verify
= manifest
.unprotected
["verify_state"].clone();
381 let verify
: Option
<SnapshotVerifyState
> = match serde_json
::from_value(verify
) {
382 Ok(verify
) => verify
,
384 eprintln
!("error parsing verification state : '{}'", err
);
389 (comment
, verify
, files
)
392 eprintln
!("error during snapshot file listing: '{}'", err
);
399 .map(|x
| BackupContent
{
400 filename
: x
.to_string(),
409 let result_item
= SnapshotListItem
{
410 backup_type
: group
.backup_type().to_string(),
411 backup_id
: group
.backup_id().to_string(),
412 backup_time
: info
.backup_dir
.backup_time(),
420 snapshots
.push(result_item
);
430 schema
: DATASTORE_SCHEMA
,
438 permission
: &Permission
::Privilege(&["datastore", "{store}"], PRIV_DATASTORE_AUDIT
| PRIV_DATASTORE_BACKUP
, true),
441 /// Get datastore status.
445 _rpcenv
: &mut dyn RpcEnvironment
,
446 ) -> Result
<StorageStatus
, Error
> {
447 let datastore
= DataStore
::lookup_datastore(&store
)?
;
448 crate::tools
::disks
::disk_usage(&datastore
.base_path())
455 schema
: DATASTORE_SCHEMA
,
458 schema
: BACKUP_TYPE_SCHEMA
,
462 schema
: BACKUP_ID_SCHEMA
,
466 schema
: BACKUP_TIME_SCHEMA
,
475 permission
: &Permission
::Privilege(&["datastore", "{store}"], PRIV_DATASTORE_READ
| PRIV_DATASTORE_BACKUP
, true), // fixme
480 /// This function can verify a single backup snapshot, all backup from a backup group,
481 /// or all backups in the datastore.
484 backup_type
: Option
<String
>,
485 backup_id
: Option
<String
>,
486 backup_time
: Option
<i64>,
487 rpcenv
: &mut dyn RpcEnvironment
,
488 ) -> Result
<Value
, Error
> {
489 let datastore
= DataStore
::lookup_datastore(&store
)?
;
493 let mut backup_dir
= None
;
494 let mut backup_group
= None
;
495 let mut worker_type
= "verify";
497 match (backup_type
, backup_id
, backup_time
) {
498 (Some(backup_type
), Some(backup_id
), Some(backup_time
)) => {
499 worker_id
= format
!("{}:{}/{}/{:08X}", store
, backup_type
, backup_id
, backup_time
);
500 let dir
= BackupDir
::new(backup_type
, backup_id
, backup_time
)?
;
501 backup_dir
= Some(dir
);
502 worker_type
= "verify_snapshot";
504 (Some(backup_type
), Some(backup_id
), None
) => {
505 worker_id
= format
!("{}:{}/{}", store
, backup_type
, backup_id
);
506 let group
= BackupGroup
::new(backup_type
, backup_id
);
507 backup_group
= Some(group
);
508 worker_type
= "verify_group";
510 (None
, None
, None
) => {
511 worker_id
= store
.clone();
513 _
=> bail
!("parameters do not specify a backup group or snapshot"),
516 let userid
: Userid
= rpcenv
.get_user().unwrap().parse()?
;
517 let to_stdout
= if rpcenv
.env_type() == RpcEnvironmentType
::CLI { true }
else { false }
;
519 let upid_str
= WorkerTask
::new_thread(
521 Some(worker_id
.clone()),
525 let verified_chunks
= Arc
::new(Mutex
::new(HashSet
::with_capacity(1024*16)));
526 let corrupt_chunks
= Arc
::new(Mutex
::new(HashSet
::with_capacity(64)));
528 let failed_dirs
= if let Some(backup_dir
) = backup_dir
{
529 let mut res
= Vec
::new();
530 if !verify_backup_dir(
536 worker
.upid().clone(),
538 res
.push(backup_dir
.to_string());
541 } else if let Some(backup_group
) = backup_group
{
542 let (_count
, failed_dirs
) = verify_backup_group(
553 verify_all_backups(datastore
, worker
.clone(), worker
.upid())?
555 if failed_dirs
.len() > 0 {
556 worker
.log("Failed to verify following snapshots:");
557 for dir
in failed_dirs
{
558 worker
.log(format
!("\t{}", dir
));
560 bail
!("verification failed - please check the log for details");
570 macro_rules
! add_common_prune_prameters
{
571 ( [ $
( $list1
:tt
)* ] ) => {
572 add_common_prune_prameters
!([$
( $list1
)* ] , [])
574 ( [ $
( $list1
:tt
)* ] , [ $
( $list2
:tt
)* ] ) => {
580 &PRUNE_SCHEMA_KEEP_DAILY
,
585 &PRUNE_SCHEMA_KEEP_HOURLY
,
590 &PRUNE_SCHEMA_KEEP_LAST
,
595 &PRUNE_SCHEMA_KEEP_MONTHLY
,
600 &PRUNE_SCHEMA_KEEP_WEEKLY
,
605 &PRUNE_SCHEMA_KEEP_YEARLY
,
612 pub const API_RETURN_SCHEMA_PRUNE
: Schema
= ArraySchema
::new(
613 "Returns the list of snapshots and a flag indicating if there are kept or removed.",
614 &PruneListItem
::API_SCHEMA
617 const API_METHOD_PRUNE
: ApiMethod
= ApiMethod
::new(
618 &ApiHandler
::Sync(&prune
),
620 "Prune the datastore.",
621 &add_common_prune_prameters
!([
622 ("backup-id", false, &BACKUP_ID_SCHEMA
),
623 ("backup-type", false, &BACKUP_TYPE_SCHEMA
),
624 ("dry-run", true, &BooleanSchema
::new(
625 "Just show what prune would do, but do not delete anything.")
629 ("store", false, &DATASTORE_SCHEMA
),
632 .returns(&API_RETURN_SCHEMA_PRUNE
)
633 .access(None
, &Permission
::Privilege(
634 &["datastore", "{store}"],
635 PRIV_DATASTORE_MODIFY
| PRIV_DATASTORE_PRUNE
,
642 rpcenv
: &mut dyn RpcEnvironment
,
643 ) -> Result
<Value
, Error
> {
645 let store
= tools
::required_string_param(¶m
, "store")?
;
646 let backup_type
= tools
::required_string_param(¶m
, "backup-type")?
;
647 let backup_id
= tools
::required_string_param(¶m
, "backup-id")?
;
649 let userid
: Userid
= rpcenv
.get_user().unwrap().parse()?
;
650 let user_info
= CachedUserInfo
::new()?
;
651 let user_privs
= user_info
.lookup_privs(&userid
, &["datastore", &store
]);
653 let dry_run
= param
["dry-run"].as_bool().unwrap_or(false);
655 let group
= BackupGroup
::new(backup_type
, backup_id
);
657 let datastore
= DataStore
::lookup_datastore(&store
)?
;
659 let allowed
= (user_privs
& PRIV_DATASTORE_MODIFY
) != 0;
660 if !allowed { check_backup_owner(&datastore, &group, &userid)?; }
662 let prune_options
= PruneOptions
{
663 keep_last
: param
["keep-last"].as_u64(),
664 keep_hourly
: param
["keep-hourly"].as_u64(),
665 keep_daily
: param
["keep-daily"].as_u64(),
666 keep_weekly
: param
["keep-weekly"].as_u64(),
667 keep_monthly
: param
["keep-monthly"].as_u64(),
668 keep_yearly
: param
["keep-yearly"].as_u64(),
671 let worker_id
= format
!("{}:{}/{}", store
, backup_type
, backup_id
);
673 let mut prune_result
= Vec
::new();
675 let list
= group
.list_backups(&datastore
.base_path())?
;
677 let mut prune_info
= compute_prune_info(list
, &prune_options
)?
;
679 prune_info
.reverse(); // delete older snapshots first
681 let keep_all
= !prune_options
.keeps_something();
684 for (info
, mut keep
) in prune_info
{
685 if keep_all { keep = true; }
687 let backup_time
= info
.backup_dir
.backup_time();
688 let group
= info
.backup_dir
.group();
690 prune_result
.push(json
!({
691 "backup-type": group
.backup_type(),
692 "backup-id": group
.backup_id(),
693 "backup-time": backup_time
,
697 return Ok(json
!(prune_result
));
701 // We use a WorkerTask just to have a task log, but run synchrounously
702 let worker
= WorkerTask
::new("prune", Some(worker_id
), Userid
::root_userid().clone(), true)?
;
705 worker
.log("No prune selection - keeping all files.");
707 worker
.log(format
!("retention options: {}", prune_options
.cli_options_string()));
708 worker
.log(format
!("Starting prune on store \"{}\" group \"{}/{}\"",
709 store
, backup_type
, backup_id
));
712 for (info
, mut keep
) in prune_info
{
713 if keep_all { keep = true; }
715 let backup_time
= info
.backup_dir
.backup_time();
716 let timestamp
= info
.backup_dir
.backup_time_string();
717 let group
= info
.backup_dir
.group();
725 if keep { "keep" }
else { "remove" }
,
730 prune_result
.push(json
!({
731 "backup-type": group
.backup_type(),
732 "backup-id": group
.backup_id(),
733 "backup-time": backup_time
,
737 if !(dry_run
|| keep
) {
738 if let Err(err
) = datastore
.remove_backup_dir(&info
.backup_dir
, false) {
741 "failed to remove dir {:?}: {}",
742 info
.backup_dir
.relative_path(), err
749 worker
.log_result(&Ok(()));
751 Ok(json
!(prune_result
))
758 schema
: DATASTORE_SCHEMA
,
766 permission
: &Permission
::Privilege(&["datastore", "{store}"], PRIV_DATASTORE_MODIFY
, false),
769 /// Start garbage collection.
770 fn start_garbage_collection(
773 rpcenv
: &mut dyn RpcEnvironment
,
774 ) -> Result
<Value
, Error
> {
776 let datastore
= DataStore
::lookup_datastore(&store
)?
;
778 println
!("Starting garbage collection on store {}", store
);
780 let to_stdout
= if rpcenv
.env_type() == RpcEnvironmentType
::CLI { true }
else { false }
;
782 let upid_str
= WorkerTask
::new_thread(
783 "garbage_collection",
785 Userid
::root_userid().clone(),
788 worker
.log(format
!("starting garbage collection on store {}", store
));
789 datastore
.garbage_collection(&*worker
, worker
.upid())
800 schema
: DATASTORE_SCHEMA
,
805 type: GarbageCollectionStatus
,
808 permission
: &Permission
::Privilege(&["datastore", "{store}"], PRIV_DATASTORE_AUDIT
, false),
811 /// Garbage collection status.
812 pub fn garbage_collection_status(
815 _rpcenv
: &mut dyn RpcEnvironment
,
816 ) -> Result
<GarbageCollectionStatus
, Error
> {
818 let datastore
= DataStore
::lookup_datastore(&store
)?
;
820 let status
= datastore
.last_gc_status();
827 description
: "List the accessible datastores.",
830 description
: "Datastore name and description.",
833 schema
: DATASTORE_SCHEMA
,
837 schema
: SINGLE_LINE_COMMENT_SCHEMA
,
843 permission
: &Permission
::Anybody
,
847 fn get_datastore_list(
850 rpcenv
: &mut dyn RpcEnvironment
,
851 ) -> Result
<Value
, Error
> {
853 let (config
, _digest
) = datastore
::config()?
;
855 let userid
: Userid
= rpcenv
.get_user().unwrap().parse()?
;
856 let user_info
= CachedUserInfo
::new()?
;
858 let mut list
= Vec
::new();
860 for (store
, (_
, data
)) in &config
.sections
{
861 let user_privs
= user_info
.lookup_privs(&userid
, &["datastore", &store
]);
862 let allowed
= (user_privs
& (PRIV_DATASTORE_AUDIT
| PRIV_DATASTORE_BACKUP
)) != 0;
864 let mut entry
= json
!({ "store": store }
);
865 if let Some(comment
) = data
["comment"].as_str() {
866 entry
["comment"] = comment
.into();
876 pub const API_METHOD_DOWNLOAD_FILE
: ApiMethod
= ApiMethod
::new(
877 &ApiHandler
::AsyncHttp(&download_file
),
879 "Download single raw file from backup snapshot.",
881 ("store", false, &DATASTORE_SCHEMA
),
882 ("backup-type", false, &BACKUP_TYPE_SCHEMA
),
883 ("backup-id", false, &BACKUP_ID_SCHEMA
),
884 ("backup-time", false, &BACKUP_TIME_SCHEMA
),
885 ("file-name", false, &BACKUP_ARCHIVE_NAME_SCHEMA
),
888 ).access(None
, &Permission
::Privilege(
889 &["datastore", "{store}"],
890 PRIV_DATASTORE_READ
| PRIV_DATASTORE_BACKUP
,
899 rpcenv
: Box
<dyn RpcEnvironment
>,
900 ) -> ApiResponseFuture
{
903 let store
= tools
::required_string_param(¶m
, "store")?
;
904 let datastore
= DataStore
::lookup_datastore(store
)?
;
906 let userid
: Userid
= rpcenv
.get_user().unwrap().parse()?
;
907 let user_info
= CachedUserInfo
::new()?
;
908 let user_privs
= user_info
.lookup_privs(&userid
, &["datastore", &store
]);
910 let file_name
= tools
::required_string_param(¶m
, "file-name")?
.to_owned();
912 let backup_type
= tools
::required_string_param(¶m
, "backup-type")?
;
913 let backup_id
= tools
::required_string_param(¶m
, "backup-id")?
;
914 let backup_time
= tools
::required_integer_param(¶m
, "backup-time")?
;
916 let backup_dir
= BackupDir
::new(backup_type
, backup_id
, backup_time
)?
;
918 let allowed
= (user_privs
& PRIV_DATASTORE_READ
) != 0;
919 if !allowed { check_backup_owner(&datastore, backup_dir.group(), &userid)?; }
921 println
!("Download {} from {} ({}/{})", file_name
, store
, backup_dir
, file_name
);
923 let mut path
= datastore
.base_path();
924 path
.push(backup_dir
.relative_path());
925 path
.push(&file_name
);
927 let file
= tokio
::fs
::File
::open(&path
)
929 .map_err(|err
| http_err
!(BAD_REQUEST
, "File open failed: {}", err
))?
;
931 let payload
= tokio_util
::codec
::FramedRead
::new(file
, tokio_util
::codec
::BytesCodec
::new())
932 .map_ok(|bytes
| hyper
::body
::Bytes
::from(bytes
.freeze()))
933 .map_err(move |err
| {
934 eprintln
!("error during streaming of '{:?}' - {}", &path
, err
);
937 let body
= Body
::wrap_stream(payload
);
939 // fixme: set other headers ?
940 Ok(Response
::builder()
941 .status(StatusCode
::OK
)
942 .header(header
::CONTENT_TYPE
, "application/octet-stream")
949 pub const API_METHOD_DOWNLOAD_FILE_DECODED
: ApiMethod
= ApiMethod
::new(
950 &ApiHandler
::AsyncHttp(&download_file_decoded
),
952 "Download single decoded file from backup snapshot. Only works if it's not encrypted.",
954 ("store", false, &DATASTORE_SCHEMA
),
955 ("backup-type", false, &BACKUP_TYPE_SCHEMA
),
956 ("backup-id", false, &BACKUP_ID_SCHEMA
),
957 ("backup-time", false, &BACKUP_TIME_SCHEMA
),
958 ("file-name", false, &BACKUP_ARCHIVE_NAME_SCHEMA
),
961 ).access(None
, &Permission
::Privilege(
962 &["datastore", "{store}"],
963 PRIV_DATASTORE_READ
| PRIV_DATASTORE_BACKUP
,
967 fn download_file_decoded(
972 rpcenv
: Box
<dyn RpcEnvironment
>,
973 ) -> ApiResponseFuture
{
976 let store
= tools
::required_string_param(¶m
, "store")?
;
977 let datastore
= DataStore
::lookup_datastore(store
)?
;
979 let userid
: Userid
= rpcenv
.get_user().unwrap().parse()?
;
980 let user_info
= CachedUserInfo
::new()?
;
981 let user_privs
= user_info
.lookup_privs(&userid
, &["datastore", &store
]);
983 let file_name
= tools
::required_string_param(¶m
, "file-name")?
.to_owned();
985 let backup_type
= tools
::required_string_param(¶m
, "backup-type")?
;
986 let backup_id
= tools
::required_string_param(¶m
, "backup-id")?
;
987 let backup_time
= tools
::required_integer_param(¶m
, "backup-time")?
;
989 let backup_dir
= BackupDir
::new(backup_type
, backup_id
, backup_time
)?
;
991 let allowed
= (user_privs
& PRIV_DATASTORE_READ
) != 0;
992 if !allowed { check_backup_owner(&datastore, backup_dir.group(), &userid)?; }
994 let (manifest
, files
) = read_backup_index(&datastore
, &backup_dir
)?
;
996 if file
.filename
== file_name
&& file
.crypt_mode
== Some(CryptMode
::Encrypt
) {
997 bail
!("cannot decode '{}' - is encrypted", file_name
);
1001 println
!("Download {} from {} ({}/{})", file_name
, store
, backup_dir
, file_name
);
1003 let mut path
= datastore
.base_path();
1004 path
.push(backup_dir
.relative_path());
1005 path
.push(&file_name
);
1007 let extension
= file_name
.rsplitn(2, '
.'
).next().unwrap();
1009 let body
= match extension
{
1011 let index
= DynamicIndexReader
::open(&path
)
1012 .map_err(|err
| format_err
!("unable to read dynamic index '{:?}' - {}", &path
, err
))?
;
1013 let (csum
, size
) = index
.compute_csum();
1014 manifest
.verify_file(&file_name
, &csum
, size
)?
;
1016 let chunk_reader
= LocalChunkReader
::new(datastore
, None
, CryptMode
::None
);
1017 let reader
= AsyncIndexReader
::new(index
, chunk_reader
);
1018 Body
::wrap_stream(AsyncReaderStream
::new(reader
)
1019 .map_err(move |err
| {
1020 eprintln
!("error during streaming of '{:?}' - {}", path
, err
);
1025 let index
= FixedIndexReader
::open(&path
)
1026 .map_err(|err
| format_err
!("unable to read fixed index '{:?}' - {}", &path
, err
))?
;
1028 let (csum
, size
) = index
.compute_csum();
1029 manifest
.verify_file(&file_name
, &csum
, size
)?
;
1031 let chunk_reader
= LocalChunkReader
::new(datastore
, None
, CryptMode
::None
);
1032 let reader
= AsyncIndexReader
::new(index
, chunk_reader
);
1033 Body
::wrap_stream(AsyncReaderStream
::with_buffer_size(reader
, 4*1024*1024)
1034 .map_err(move |err
| {
1035 eprintln
!("error during streaming of '{:?}' - {}", path
, err
);
1040 let file
= std
::fs
::File
::open(&path
)
1041 .map_err(|err
| http_err
!(BAD_REQUEST
, "File open failed: {}", err
))?
;
1043 // FIXME: load full blob to verify index checksum?
1046 WrappedReaderStream
::new(DataBlobReader
::new(file
, None
)?
)
1047 .map_err(move |err
| {
1048 eprintln
!("error during streaming of '{:?}' - {}", path
, err
);
1054 bail
!("cannot download '{}' files", extension
);
1058 // fixme: set other headers ?
1059 Ok(Response
::builder()
1060 .status(StatusCode
::OK
)
1061 .header(header
::CONTENT_TYPE
, "application/octet-stream")
1068 pub const API_METHOD_UPLOAD_BACKUP_LOG
: ApiMethod
= ApiMethod
::new(
1069 &ApiHandler
::AsyncHttp(&upload_backup_log
),
1071 "Upload the client backup log file into a backup snapshot ('client.log.blob').",
1073 ("store", false, &DATASTORE_SCHEMA
),
1074 ("backup-type", false, &BACKUP_TYPE_SCHEMA
),
1075 ("backup-id", false, &BACKUP_ID_SCHEMA
),
1076 ("backup-time", false, &BACKUP_TIME_SCHEMA
),
1080 Some("Only the backup creator/owner is allowed to do this."),
1081 &Permission
::Privilege(&["datastore", "{store}"], PRIV_DATASTORE_BACKUP
, false)
1084 fn upload_backup_log(
1089 rpcenv
: Box
<dyn RpcEnvironment
>,
1090 ) -> ApiResponseFuture
{
1093 let store
= tools
::required_string_param(¶m
, "store")?
;
1094 let datastore
= DataStore
::lookup_datastore(store
)?
;
1096 let file_name
= CLIENT_LOG_BLOB_NAME
;
1098 let backup_type
= tools
::required_string_param(¶m
, "backup-type")?
;
1099 let backup_id
= tools
::required_string_param(¶m
, "backup-id")?
;
1100 let backup_time
= tools
::required_integer_param(¶m
, "backup-time")?
;
1102 let backup_dir
= BackupDir
::new(backup_type
, backup_id
, backup_time
)?
;
1104 let userid
: Userid
= rpcenv
.get_user().unwrap().parse()?
;
1105 check_backup_owner(&datastore
, backup_dir
.group(), &userid
)?
;
1107 let mut path
= datastore
.base_path();
1108 path
.push(backup_dir
.relative_path());
1109 path
.push(&file_name
);
1112 bail
!("backup already contains a log.");
1115 println
!("Upload backup log to {}/{}/{}/{}/{}", store
,
1116 backup_type
, backup_id
, backup_dir
.backup_time_string(), file_name
);
1119 .map_err(Error
::from
)
1120 .try_fold(Vec
::new(), |mut acc
, chunk
| {
1121 acc
.extend_from_slice(&*chunk
);
1122 future
::ok
::<_
, Error
>(acc
)
1126 // always verify blob/CRC at server side
1127 let blob
= DataBlob
::load_from_reader(&mut &data
[..])?
;
1129 replace_file(&path
, blob
.raw_data(), CreateOptions
::new())?
;
1131 // fixme: use correct formatter
1132 Ok(crate::server
::formatter
::json_response(Ok(Value
::Null
)))
1140 schema
: DATASTORE_SCHEMA
,
1143 schema
: BACKUP_TYPE_SCHEMA
,
1146 schema
: BACKUP_ID_SCHEMA
,
1149 schema
: BACKUP_TIME_SCHEMA
,
1152 description
: "Base64 encoded path.",
1158 permission
: &Permission
::Privilege(&["datastore", "{store}"], PRIV_DATASTORE_READ
| PRIV_DATASTORE_BACKUP
, true),
1161 /// Get the entries of the given path of the catalog
1164 backup_type
: String
,
1170 rpcenv
: &mut dyn RpcEnvironment
,
1171 ) -> Result
<Value
, Error
> {
1172 let datastore
= DataStore
::lookup_datastore(&store
)?
;
1174 let userid
: Userid
= rpcenv
.get_user().unwrap().parse()?
;
1175 let user_info
= CachedUserInfo
::new()?
;
1176 let user_privs
= user_info
.lookup_privs(&userid
, &["datastore", &store
]);
1178 let backup_dir
= BackupDir
::new(backup_type
, backup_id
, backup_time
)?
;
1180 let allowed
= (user_privs
& PRIV_DATASTORE_READ
) != 0;
1181 if !allowed { check_backup_owner(&datastore, backup_dir.group(), &userid)?; }
1183 let file_name
= CATALOG_NAME
;
1185 let (manifest
, files
) = read_backup_index(&datastore
, &backup_dir
)?
;
1187 if file
.filename
== file_name
&& file
.crypt_mode
== Some(CryptMode
::Encrypt
) {
1188 bail
!("cannot decode '{}' - is encrypted", file_name
);
1192 let mut path
= datastore
.base_path();
1193 path
.push(backup_dir
.relative_path());
1194 path
.push(file_name
);
1196 let index
= DynamicIndexReader
::open(&path
)
1197 .map_err(|err
| format_err
!("unable to read dynamic index '{:?}' - {}", &path
, err
))?
;
1199 let (csum
, size
) = index
.compute_csum();
1200 manifest
.verify_file(&file_name
, &csum
, size
)?
;
1202 let chunk_reader
= LocalChunkReader
::new(datastore
, None
, CryptMode
::None
);
1203 let reader
= BufferedDynamicReader
::new(index
, chunk_reader
);
1205 let mut catalog_reader
= CatalogReader
::new(reader
);
1206 let mut current
= catalog_reader
.root()?
;
1207 let mut components
= vec
![];
1210 if filepath
!= "root" {
1211 components
= base64
::decode(filepath
)?
;
1212 if components
.len() > 0 && components
[0] == '
/'
as u8 {
1213 components
.remove(0);
1215 for component
in components
.split(|c
| *c
== '
/'
as u8) {
1216 if let Some(entry
) = catalog_reader
.lookup(¤t
, component
)?
{
1219 bail
!("path {:?} not found in catalog", &String
::from_utf8_lossy(&components
));
1224 let mut res
= Vec
::new();
1226 for direntry
in catalog_reader
.read_dir(¤t
)?
{
1227 let mut components
= components
.clone();
1228 components
.push('
/'
as u8);
1229 components
.extend(&direntry
.name
);
1230 let path
= base64
::encode(components
);
1231 let text
= String
::from_utf8_lossy(&direntry
.name
);
1232 let mut entry
= json
!({
1235 "type": CatalogEntryType
::from(&direntry
.attr
).to_string(),
1238 match direntry
.attr
{
1239 DirEntryAttribute
::Directory { start: _ }
=> {
1240 entry
["leaf"] = false.into();
1242 DirEntryAttribute
::File { size, mtime }
=> {
1243 entry
["size"] = size
.into();
1244 entry
["mtime"] = mtime
.into();
1254 fn recurse_files
<'a
, T
, W
>(
1255 zip
: &'a
mut ZipEncoder
<W
>,
1256 decoder
: &'a
mut Accessor
<T
>,
1259 ) -> Pin
<Box
<dyn Future
<Output
= Result
<(), Error
>> + Send
+ 'a
>>
1261 T
: Clone
+ pxar
::accessor
::ReadAt
+ Unpin
+ Send
+ Sync
+ '
static,
1262 W
: tokio
::io
::AsyncWrite
+ Unpin
+ Send
+ '
static,
1264 Box
::pin(async
move {
1265 let metadata
= file
.entry().metadata();
1266 let path
= file
.entry().path().strip_prefix(&prefix
)?
.to_path_buf();
1269 EntryKind
::File { .. }
=> {
1270 let entry
= ZipEntry
::new(
1272 metadata
.stat
.mtime
.secs
,
1273 metadata
.stat
.mode
as u16,
1276 zip
.add_entry(entry
, Some(file
.contents().await?
))
1278 .map_err(|err
| format_err
!("could not send file entry: {}", err
))?
;
1280 EntryKind
::Hardlink(_
) => {
1281 let realfile
= decoder
.follow_hardlink(&file
).await?
;
1282 let entry
= ZipEntry
::new(
1284 metadata
.stat
.mtime
.secs
,
1285 metadata
.stat
.mode
as u16,
1288 zip
.add_entry(entry
, Some(realfile
.contents().await?
))
1290 .map_err(|err
| format_err
!("could not send file entry: {}", err
))?
;
1292 EntryKind
::Directory
=> {
1293 let dir
= file
.enter_directory().await?
;
1294 let mut readdir
= dir
.read_dir();
1295 let entry
= ZipEntry
::new(
1297 metadata
.stat
.mtime
.secs
,
1298 metadata
.stat
.mode
as u16,
1301 zip
.add_entry
::<FileContents
<T
>>(entry
, None
).await?
;
1302 while let Some(entry
) = readdir
.next().await
{
1303 let entry
= entry?
.decode_entry().await?
;
1304 recurse_files(zip
, decoder
, prefix
, entry
).await?
;
1307 _
=> {}
// ignore all else
1315 pub const API_METHOD_PXAR_FILE_DOWNLOAD
: ApiMethod
= ApiMethod
::new(
1316 &ApiHandler
::AsyncHttp(&pxar_file_download
),
1318 "Download single file from pxar file of a backup snapshot. Only works if it's not encrypted.",
1320 ("store", false, &DATASTORE_SCHEMA
),
1321 ("backup-type", false, &BACKUP_TYPE_SCHEMA
),
1322 ("backup-id", false, &BACKUP_ID_SCHEMA
),
1323 ("backup-time", false, &BACKUP_TIME_SCHEMA
),
1324 ("filepath", false, &StringSchema
::new("Base64 encoded path").schema()),
1327 ).access(None
, &Permission
::Privilege(
1328 &["datastore", "{store}"],
1329 PRIV_DATASTORE_READ
| PRIV_DATASTORE_BACKUP
,
1333 fn pxar_file_download(
1338 rpcenv
: Box
<dyn RpcEnvironment
>,
1339 ) -> ApiResponseFuture
{
1342 let store
= tools
::required_string_param(¶m
, "store")?
;
1343 let datastore
= DataStore
::lookup_datastore(&store
)?
;
1345 let userid
: Userid
= rpcenv
.get_user().unwrap().parse()?
;
1346 let user_info
= CachedUserInfo
::new()?
;
1347 let user_privs
= user_info
.lookup_privs(&userid
, &["datastore", &store
]);
1349 let filepath
= tools
::required_string_param(¶m
, "filepath")?
.to_owned();
1351 let backup_type
= tools
::required_string_param(¶m
, "backup-type")?
;
1352 let backup_id
= tools
::required_string_param(¶m
, "backup-id")?
;
1353 let backup_time
= tools
::required_integer_param(¶m
, "backup-time")?
;
1355 let backup_dir
= BackupDir
::new(backup_type
, backup_id
, backup_time
)?
;
1357 let allowed
= (user_privs
& PRIV_DATASTORE_READ
) != 0;
1358 if !allowed { check_backup_owner(&datastore, backup_dir.group(), &userid)?; }
1360 let mut components
= base64
::decode(&filepath
)?
;
1361 if components
.len() > 0 && components
[0] == '
/'
as u8 {
1362 components
.remove(0);
1365 let mut split
= components
.splitn(2, |c
| *c
== '
/'
as u8);
1366 let pxar_name
= std
::str::from_utf8(split
.next().unwrap())?
;
1367 let file_path
= split
.next().ok_or(format_err
!("filepath looks strange '{}'", filepath
))?
;
1368 let (manifest
, files
) = read_backup_index(&datastore
, &backup_dir
)?
;
1370 if file
.filename
== pxar_name
&& file
.crypt_mode
== Some(CryptMode
::Encrypt
) {
1371 bail
!("cannot decode '{}' - is encrypted", pxar_name
);
1375 let mut path
= datastore
.base_path();
1376 path
.push(backup_dir
.relative_path());
1377 path
.push(pxar_name
);
1379 let index
= DynamicIndexReader
::open(&path
)
1380 .map_err(|err
| format_err
!("unable to read dynamic index '{:?}' - {}", &path
, err
))?
;
1382 let (csum
, size
) = index
.compute_csum();
1383 manifest
.verify_file(&pxar_name
, &csum
, size
)?
;
1385 let chunk_reader
= LocalChunkReader
::new(datastore
, None
, CryptMode
::None
);
1386 let reader
= BufferedDynamicReader
::new(index
, chunk_reader
);
1387 let archive_size
= reader
.archive_size();
1388 let reader
= LocalDynamicReadAt
::new(reader
);
1390 let decoder
= Accessor
::new(reader
, archive_size
).await?
;
1391 let root
= decoder
.open_root().await?
;
1393 .lookup(OsStr
::from_bytes(file_path
)).await?
1394 .ok_or(format_err
!("error opening '{:?}'", file_path
))?
;
1396 let body
= match file
.kind() {
1397 EntryKind
::File { .. }
=> Body
::wrap_stream(
1398 AsyncReaderStream
::new(file
.contents().await?
).map_err(move |err
| {
1399 eprintln
!("error during streaming of file '{:?}' - {}", filepath
, err
);
1403 EntryKind
::Hardlink(_
) => Body
::wrap_stream(
1404 AsyncReaderStream
::new(decoder
.follow_hardlink(&file
).await?
.contents().await?
)
1405 .map_err(move |err
| {
1407 "error during streaming of hardlink '{:?}' - {}",
1413 EntryKind
::Directory
=> {
1414 let (sender
, receiver
) = tokio
::sync
::mpsc
::channel(100);
1415 let mut prefix
= PathBuf
::new();
1416 let mut components
= file
.entry().path().components();
1417 components
.next_back(); // discar last
1418 for comp
in components
{
1422 let channelwriter
= AsyncChannelWriter
::new(sender
, 1024 * 1024);
1424 crate::server
::spawn_internal_task(async
move {
1425 let mut zipencoder
= ZipEncoder
::new(channelwriter
);
1426 let mut decoder
= decoder
;
1427 recurse_files(&mut zipencoder
, &mut decoder
, &prefix
, file
)
1429 .map_err(|err
| eprintln
!("error during creating of zip: {}", err
))?
;
1434 .map_err(|err
| eprintln
!("error during finishing of zip: {}", err
))
1437 Body
::wrap_stream(receiver
.map_err(move |err
| {
1438 eprintln
!("error during streaming of zip '{:?}' - {}", filepath
, err
);
1442 other
=> bail
!("cannot download file of type {:?}", other
),
1445 // fixme: set other headers ?
1446 Ok(Response
::builder()
1447 .status(StatusCode
::OK
)
1448 .header(header
::CONTENT_TYPE
, "application/octet-stream")
1458 schema
: DATASTORE_SCHEMA
,
1461 type: RRDTimeFrameResolution
,
1469 permission
: &Permission
::Privilege(&["datastore", "{store}"], PRIV_DATASTORE_AUDIT
| PRIV_DATASTORE_BACKUP
, true),
1472 /// Read datastore stats
1475 timeframe
: RRDTimeFrameResolution
,
1478 ) -> Result
<Value
, Error
> {
1480 create_value_from_rrd(
1481 &format
!("datastore/{}", store
),
1484 "read_ios", "read_bytes",
1485 "write_ios", "write_bytes",
1497 schema
: DATASTORE_SCHEMA
,
1500 schema
: BACKUP_TYPE_SCHEMA
,
1503 schema
: BACKUP_ID_SCHEMA
,
1506 schema
: BACKUP_TIME_SCHEMA
,
1511 permission
: &Permission
::Privilege(&["datastore", "{store}"], PRIV_DATASTORE_READ
| PRIV_DATASTORE_BACKUP
, true),
1514 /// Get "notes" for a specific backup
1517 backup_type
: String
,
1520 rpcenv
: &mut dyn RpcEnvironment
,
1521 ) -> Result
<String
, Error
> {
1522 let datastore
= DataStore
::lookup_datastore(&store
)?
;
1524 let userid
: Userid
= rpcenv
.get_user().unwrap().parse()?
;
1525 let user_info
= CachedUserInfo
::new()?
;
1526 let user_privs
= user_info
.lookup_privs(&userid
, &["datastore", &store
]);
1528 let backup_dir
= BackupDir
::new(backup_type
, backup_id
, backup_time
)?
;
1530 let allowed
= (user_privs
& PRIV_DATASTORE_READ
) != 0;
1531 if !allowed { check_backup_owner(&datastore, backup_dir.group(), &userid)?; }
1533 let (manifest
, _
) = datastore
.load_manifest(&backup_dir
)?
;
1535 let notes
= manifest
.unprotected
["notes"]
1539 Ok(String
::from(notes
))
1546 schema
: DATASTORE_SCHEMA
,
1549 schema
: BACKUP_TYPE_SCHEMA
,
1552 schema
: BACKUP_ID_SCHEMA
,
1555 schema
: BACKUP_TIME_SCHEMA
,
1558 description
: "A multiline text.",
1563 permission
: &Permission
::Privilege(&["datastore", "{store}"], PRIV_DATASTORE_MODIFY
, true),
1566 /// Set "notes" for a specific backup
1569 backup_type
: String
,
1573 rpcenv
: &mut dyn RpcEnvironment
,
1574 ) -> Result
<(), Error
> {
1575 let datastore
= DataStore
::lookup_datastore(&store
)?
;
1577 let userid
: Userid
= rpcenv
.get_user().unwrap().parse()?
;
1578 let user_info
= CachedUserInfo
::new()?
;
1579 let user_privs
= user_info
.lookup_privs(&userid
, &["datastore", &store
]);
1581 let backup_dir
= BackupDir
::new(backup_type
, backup_id
, backup_time
)?
;
1583 let allowed
= (user_privs
& PRIV_DATASTORE_READ
) != 0;
1584 if !allowed { check_backup_owner(&datastore, backup_dir.group(), &userid)?; }
1586 datastore
.update_manifest(&backup_dir
,|manifest
| {
1587 manifest
.unprotected
["notes"] = notes
.into();
1588 }).map_err(|err
| format_err
!("unable to update manifest blob - {}", err
))?
;
1597 schema
: DATASTORE_SCHEMA
,
1600 schema
: BACKUP_TYPE_SCHEMA
,
1603 schema
: BACKUP_ID_SCHEMA
,
1611 permission
: &Permission
::Privilege(&["datastore", "{store}"], PRIV_DATASTORE_MODIFY
, true),
1614 /// Change owner of a backup group
1615 fn set_backup_owner(
1617 backup_type
: String
,
1620 _rpcenv
: &mut dyn RpcEnvironment
,
1621 ) -> Result
<(), Error
> {
1623 let datastore
= DataStore
::lookup_datastore(&store
)?
;
1625 let backup_group
= BackupGroup
::new(backup_type
, backup_id
);
1627 let user_info
= CachedUserInfo
::new()?
;
1629 if !user_info
.is_active_user(&new_owner
) {
1630 bail
!("user '{}' is inactive or non-existent", new_owner
);
1633 datastore
.set_owner(&backup_group
, &new_owner
, true)?
;
1639 const DATASTORE_INFO_SUBDIRS
: SubdirMap
= &[
1643 .get(&API_METHOD_CATALOG
)
1648 .post(&API_METHOD_SET_BACKUP_OWNER
)
1653 .download(&API_METHOD_DOWNLOAD_FILE
)
1658 .download(&API_METHOD_DOWNLOAD_FILE_DECODED
)
1663 .get(&API_METHOD_LIST_SNAPSHOT_FILES
)
1668 .get(&API_METHOD_GARBAGE_COLLECTION_STATUS
)
1669 .post(&API_METHOD_START_GARBAGE_COLLECTION
)
1674 .get(&API_METHOD_LIST_GROUPS
)
1679 .get(&API_METHOD_GET_NOTES
)
1680 .put(&API_METHOD_SET_NOTES
)
1685 .post(&API_METHOD_PRUNE
)
1688 "pxar-file-download",
1690 .download(&API_METHOD_PXAR_FILE_DOWNLOAD
)
1695 .get(&API_METHOD_GET_RRD_STATS
)
1700 .get(&API_METHOD_LIST_SNAPSHOTS
)
1701 .delete(&API_METHOD_DELETE_SNAPSHOT
)
1706 .get(&API_METHOD_STATUS
)
1709 "upload-backup-log",
1711 .upload(&API_METHOD_UPLOAD_BACKUP_LOG
)
1716 .post(&API_METHOD_VERIFY
)
1720 const DATASTORE_INFO_ROUTER
: Router
= Router
::new()
1721 .get(&list_subdirs_api_method
!(DATASTORE_INFO_SUBDIRS
))
1722 .subdirs(DATASTORE_INFO_SUBDIRS
);
1725 pub const ROUTER
: Router
= Router
::new()
1726 .get(&API_METHOD_GET_DATASTORE_LIST
)
1727 .match_all("store", &DATASTORE_INFO_ROUTER
);