]> git.proxmox.com Git - proxmox-backup.git/blobdiff - src/api2/admin/datastore.rs
move required_X_param to pbs_tools::json
[proxmox-backup.git] / src / api2 / admin / datastore.rs
index 09cde15e955cb62167b24a2a12b6583aa27d6452..d1778fd94cf95f9d0c319c35e3ed2c115fdcc105 100644 (file)
@@ -1,9 +1,9 @@
+//! Datastore Management
+
 use std::collections::HashSet;
 use std::ffi::OsStr;
 use std::os::unix::ffi::OsStrExt;
-use std::sync::{Arc, Mutex};
-use std::path::{Path, PathBuf};
-use std::pin::Pin;
+use std::path::PathBuf;
 
 use anyhow::{bail, format_err, Error};
 use futures::*;
@@ -16,26 +16,28 @@ use proxmox::api::{
     api, ApiResponseFuture, ApiHandler, ApiMethod, Router,
     RpcEnvironment, RpcEnvironmentType, Permission
 };
-use proxmox::api::router::{ReturnType, SubdirMap};
+use proxmox::api::router::SubdirMap;
 use proxmox::api::schema::*;
-use proxmox::tools::fs::{replace_file, CreateOptions};
+use proxmox::tools::fs::{
+    file_read_firstline, file_read_optional_string, replace_file, CreateOptions,
+};
 use proxmox::{http_err, identity, list_subdirs_api_method, sortable};
 
-use pxar::accessor::aio::{Accessor, FileContents, FileEntry};
+use pxar::accessor::aio::Accessor;
 use pxar::EntryKind;
 
+use pbs_client::pxar::create_zip;
+use pbs_tools::json::{required_integer_param, required_string_param};
+
 use crate::api2::types::*;
 use crate::api2::node::rrd::create_value_from_rrd;
+use crate::api2::helpers;
 use crate::backup::*;
 use crate::config::datastore;
 use crate::config::cached_user_info::CachedUserInfo;
 
 use crate::server::{jobstate::Job, WorkerTask};
-use crate::tools::{
-    self,
-    zip::{ZipEncoder, ZipEntry},
-    AsyncChannelWriter, AsyncReaderStream, WrappedReaderStream,
-};
+use crate::tools::{AsyncChannelWriter, AsyncReaderStream, WrappedReaderStream};
 
 use crate::config::acl::{
     PRIV_DATASTORE_AUDIT,
@@ -46,6 +48,15 @@ use crate::config::acl::{
     PRIV_DATASTORE_VERIFY,
 };
 
+const GROUP_NOTES_FILE_NAME: &str = "notes";
+
+fn get_group_note_path(store: &DataStore, group: &BackupGroup) -> PathBuf {
+    let mut note_path = store.base_path();
+    note_path.push(group.group_path());
+    note_path.push(GROUP_NOTES_FILE_NAME);
+    note_path
+}
+
 fn check_priv_or_backup_owner(
     store: &DataStore,
     group: &BackupGroup,
@@ -62,18 +73,6 @@ fn check_priv_or_backup_owner(
     Ok(())
 }
 
-fn check_backup_owner(
-    owner: &Authid,
-    auth_id: &Authid,
-) -> Result<(), Error> {
-    let correct_owner = owner == auth_id
-        || (owner.is_token() && &Authid::from(owner.user().clone()) == auth_id);
-    if !correct_owner {
-        bail!("backup owner check failed ({} != {})", auth_id, owner);
-    }
-    Ok(())
-}
-
 fn read_backup_index(
     store: &DataStore,
     backup_dir: &BackupDir,
@@ -204,6 +203,9 @@ pub fn list_groups(
                 })
                 .to_owned();
 
+            let note_path = get_group_note_path(&datastore, &group);
+            let comment = file_read_firstline(&note_path).ok();
+
             group_info.push(GroupListItem {
                 backup_type: group.backup_type().to_string(),
                 backup_id: group.backup_id().to_string(),
@@ -211,6 +213,7 @@ pub fn list_groups(
                 owner: Some(owner),
                 backup_count,
                 files: last_backup.files,
+                comment,
             });
 
             group_info
@@ -219,6 +222,48 @@ pub fn list_groups(
     Ok(group_info)
 }
 
+#[api(
+    input: {
+        properties: {
+            store: {
+                schema: DATASTORE_SCHEMA,
+            },
+            "backup-type": {
+                schema: BACKUP_TYPE_SCHEMA,
+            },
+            "backup-id": {
+                schema: BACKUP_ID_SCHEMA,
+            },
+        },
+    },
+    access: {
+        permission: &Permission::Privilege(
+            &["datastore", "{store}"],
+            PRIV_DATASTORE_MODIFY| PRIV_DATASTORE_PRUNE,
+            true),
+    },
+)]
+/// Delete backup group including all snapshots.
+pub fn delete_group(
+    store: String,
+    backup_type: String,
+    backup_id: String,
+    _info: &ApiMethod,
+    rpcenv: &mut dyn RpcEnvironment,
+) -> Result<Value, Error> {
+
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+
+    let group = BackupGroup::new(backup_type, backup_id);
+    let datastore = DataStore::lookup_datastore(&store)?;
+
+    check_priv_or_backup_owner(&datastore, &group, &auth_id, PRIV_DATASTORE_MODIFY)?;
+
+    datastore.remove_backup_group(&group)?;
+
+    Ok(Value::Null)
+}
+
 #[api(
     input: {
         properties: {
@@ -299,7 +344,7 @@ pub fn list_snapshot_files(
     },
 )]
 /// Delete backup snapshot.
-fn delete_snapshot(
+pub fn delete_snapshot(
     store: String,
     backup_type: String,
     backup_id: String,
@@ -604,6 +649,14 @@ pub fn status(
                 schema: BACKUP_ID_SCHEMA,
                 optional: true,
             },
+            "ignore-verified": {
+                schema: IGNORE_VERIFIED_BACKUPS_SCHEMA,
+                optional: true,
+            },
+            "outdated-after": {
+                schema: VERIFICATION_OUTDATED_AFTER_SCHEMA,
+                optional: true,
+            },
             "backup-time": {
                 schema: BACKUP_TIME_SCHEMA,
                 optional: true,
@@ -626,9 +679,12 @@ pub fn verify(
     backup_type: Option<String>,
     backup_id: Option<String>,
     backup_time: Option<i64>,
+    ignore_verified: Option<bool>,
+    outdated_after: Option<i64>,
     rpcenv: &mut dyn RpcEnvironment,
 ) -> Result<Value, Error> {
     let datastore = DataStore::lookup_datastore(&store)?;
+    let ignore_verified = ignore_verified.unwrap_or(true);
 
     let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
     let worker_id;
@@ -670,33 +726,29 @@ pub fn verify(
         auth_id.clone(),
         to_stdout,
         move |worker| {
-            let verified_chunks = Arc::new(Mutex::new(HashSet::with_capacity(1024*16)));
-            let corrupt_chunks = Arc::new(Mutex::new(HashSet::with_capacity(64)));
-
+            let verify_worker = crate::backup::VerifyWorker::new(worker.clone(), datastore);
             let failed_dirs = if let Some(backup_dir) = backup_dir {
                 let mut res = Vec::new();
                 if !verify_backup_dir(
-                    datastore,
+                    &verify_worker,
                     &backup_dir,
-                    verified_chunks,
-                    corrupt_chunks,
-                    worker.clone(),
                     worker.upid().clone(),
-                    None,
+                    Some(&move |manifest| {
+                        verify_filter(ignore_verified, outdated_after, manifest)
+                    }),
                 )? {
                     res.push(backup_dir.to_string());
                 }
                 res
             } else if let Some(backup_group) = backup_group {
                 let failed_dirs = verify_backup_group(
-                    datastore,
+                    &verify_worker,
                     &backup_group,
-                    verified_chunks,
-                    corrupt_chunks,
                     &mut StoreProgress::new(1),
-                    worker.clone(),
                     worker.upid(),
-                    None,
+                    Some(&move |manifest| {
+                        verify_filter(ignore_verified, outdated_after, manifest)
+                    }),
                 )?;
                 failed_dirs
             } else {
@@ -709,7 +761,14 @@ pub fn verify(
                     None
                 };
 
-                verify_all_backups(datastore, worker.clone(), worker.upid(), owner, None)?
+                verify_all_backups(
+                    &verify_worker,
+                    worker.upid(),
+                    owner,
+                    Some(&move |manifest| {
+                        verify_filter(ignore_verified, outdated_after, manifest)
+                    }),
+                )?
             };
             if !failed_dirs.is_empty() {
                 worker.log("Failed to verify the following snapshots/groups:");
@@ -725,106 +784,61 @@ pub fn verify(
     Ok(json!(upid_str))
 }
 
-#[macro_export]
-macro_rules! add_common_prune_prameters {
-    ( [ $( $list1:tt )* ] ) => {
-        add_common_prune_prameters!([$( $list1 )* ] ,  [])
-    };
-    ( [ $( $list1:tt )* ] ,  [ $( $list2:tt )* ] ) => {
-        [
-            $( $list1 )*
-            (
-                "keep-daily",
-                true,
-                &PRUNE_SCHEMA_KEEP_DAILY,
-            ),
-            (
-                "keep-hourly",
-                true,
-                &PRUNE_SCHEMA_KEEP_HOURLY,
-            ),
-            (
-                "keep-last",
-                true,
-                &PRUNE_SCHEMA_KEEP_LAST,
-            ),
-            (
-                "keep-monthly",
-                true,
-                &PRUNE_SCHEMA_KEEP_MONTHLY,
-            ),
-            (
-                "keep-weekly",
-                true,
-                &PRUNE_SCHEMA_KEEP_WEEKLY,
-            ),
-            (
-                "keep-yearly",
-                true,
-                &PRUNE_SCHEMA_KEEP_YEARLY,
-            ),
-            $( $list2 )*
-        ]
-    }
-}
-
-pub const API_RETURN_SCHEMA_PRUNE: Schema = ArraySchema::new(
-    "Returns the list of snapshots and a flag indicating if there are kept or removed.",
-    &PruneListItem::API_SCHEMA
-).schema();
-
-pub const API_METHOD_PRUNE: ApiMethod = ApiMethod::new(
-    &ApiHandler::Sync(&prune),
-    &ObjectSchema::new(
-        "Prune the datastore.",
-        &add_common_prune_prameters!([
-            ("backup-id", false, &BACKUP_ID_SCHEMA),
-            ("backup-type", false, &BACKUP_TYPE_SCHEMA),
-            ("dry-run", true, &BooleanSchema::new(
-                "Just show what prune would do, but do not delete anything.")
-             .schema()
-            ),
-        ],[
-            ("store", false, &DATASTORE_SCHEMA),
-        ])
-    ))
-    .returns(ReturnType::new(false, &API_RETURN_SCHEMA_PRUNE))
-    .access(None, &Permission::Privilege(
-    &["datastore", "{store}"],
-    PRIV_DATASTORE_MODIFY | PRIV_DATASTORE_PRUNE,
-    true)
-);
-
-fn prune(
-    param: Value,
-    _info: &ApiMethod,
+#[api(
+    input: {
+        properties: {
+            "backup-id": {
+                schema: BACKUP_ID_SCHEMA,
+            },
+            "backup-type": {
+                schema: BACKUP_TYPE_SCHEMA,
+            },
+            "dry-run": {
+                optional: true,
+                type: bool,
+                default: false,
+                description: "Just show what prune would do, but do not delete anything.",
+            },
+            "prune-options": {
+                type: PruneOptions,
+                flatten: true,
+            },
+            store: {
+                schema: DATASTORE_SCHEMA,
+            },
+        },
+    },
+    returns: {
+        type: Array,
+        description: "Returns the list of snapshots and a flag indicating if there are kept or removed.",
+        items: {
+            type: PruneListItem,
+        },
+    },
+    access: {
+        permission: &Permission::Privilege(&["datastore", "{store}"], PRIV_DATASTORE_MODIFY | PRIV_DATASTORE_PRUNE, true),
+    },
+)]
+/// Prune a group on the datastore
+pub fn prune(
+    backup_id: String,
+    backup_type: String,
+    dry_run: bool,
+    prune_options: PruneOptions,
+    store: String,
+    _param: Value,
     rpcenv: &mut dyn RpcEnvironment,
 ) -> Result<Value, Error> {
 
-    let store = tools::required_string_param(&param, "store")?;
-    let backup_type = tools::required_string_param(&param, "backup-type")?;
-    let backup_id = tools::required_string_param(&param, "backup-id")?;
-
     let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
 
-    let dry_run = param["dry-run"].as_bool().unwrap_or(false);
-
-    let group = BackupGroup::new(backup_type, backup_id);
+    let group = BackupGroup::new(&backup_type, &backup_id);
 
     let datastore = DataStore::lookup_datastore(&store)?;
 
     check_priv_or_backup_owner(&datastore, &group, &auth_id, PRIV_DATASTORE_MODIFY)?;
 
-    let prune_options = PruneOptions {
-        keep_last: param["keep-last"].as_u64(),
-        keep_hourly: param["keep-hourly"].as_u64(),
-        keep_daily: param["keep-daily"].as_u64(),
-        keep_weekly: param["keep-weekly"].as_u64(),
-        keep_monthly: param["keep-monthly"].as_u64(),
-        keep_yearly: param["keep-yearly"].as_u64(),
-    };
-
-    let worker_id = format!("{}:{}/{}", store, backup_type, backup_id);
+    let worker_id = format!("{}:{}/{}", store, &backup_type, &backup_id);
 
     let mut prune_result = Vec::new();
 
@@ -907,6 +921,62 @@ fn prune(
     Ok(json!(prune_result))
 }
 
+#[api(
+    input: {
+        properties: {
+            "dry-run": {
+                optional: true,
+                type: bool,
+                default: false,
+                description: "Just show what prune would do, but do not delete anything.",
+            },
+            "prune-options": {
+                type: PruneOptions,
+                flatten: true,
+            },
+            store: {
+                schema: DATASTORE_SCHEMA,
+            },
+        },
+    },
+    returns: {
+        schema: UPID_SCHEMA,
+    },
+    access: {
+        permission: &Permission::Privilege(&["datastore", "{store}"], PRIV_DATASTORE_MODIFY | PRIV_DATASTORE_PRUNE, true),
+    },
+)]
+/// Prune the datastore
+pub fn prune_datastore(
+    dry_run: bool,
+    prune_options: PruneOptions,
+    store: String,
+    _param: Value,
+    rpcenv: &mut dyn RpcEnvironment,
+) -> Result<String, Error> {
+
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+
+    let datastore = DataStore::lookup_datastore(&store)?;
+
+    let upid_str = WorkerTask::new_thread(
+        "prune",
+        Some(store.clone()),
+        auth_id.clone(),
+        false,
+        move |worker| crate::server::prune_datastore(
+            worker.clone(),
+            auth_id,
+            prune_options,
+            &store,
+            datastore,
+            dry_run
+        ),
+    )?;
+
+    Ok(upid_str)
+}
+
 #[api(
     input: {
         properties: {
@@ -923,7 +993,7 @@ fn prune(
     },
 )]
 /// Start garbage collection.
-fn start_garbage_collection(
+pub fn start_garbage_collection(
     store: String,
     _info: &ApiMethod,
     rpcenv: &mut dyn RpcEnvironment,
@@ -983,7 +1053,7 @@ pub fn garbage_collection_status(
     },
 )]
 /// Datastore list
-fn get_datastore_list(
+pub fn get_datastore_list(
     _param: Value,
     _info: &ApiMethod,
     rpcenv: &mut dyn RpcEnvironment,
@@ -1031,7 +1101,7 @@ pub const API_METHOD_DOWNLOAD_FILE: ApiMethod = ApiMethod::new(
     true)
 );
 
-fn download_file(
+pub fn download_file(
     _parts: Parts,
     _req_body: Body,
     param: Value,
@@ -1040,16 +1110,16 @@ fn download_file(
 ) -> ApiResponseFuture {
 
     async move {
-        let store = tools::required_string_param(&param, "store")?;
+        let store = required_string_param(&param, "store")?;
         let datastore = DataStore::lookup_datastore(store)?;
 
         let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
 
-        let file_name = tools::required_string_param(&param, "file-name")?.to_owned();
+        let file_name = required_string_param(&param, "file-name")?.to_owned();
 
-        let backup_type = tools::required_string_param(&param, "backup-type")?;
-        let backup_id = tools::required_string_param(&param, "backup-id")?;
-        let backup_time = tools::required_integer_param(&param, "backup-time")?;
+        let backup_type = required_string_param(&param, "backup-type")?;
+        let backup_id = required_string_param(&param, "backup-id")?;
+        let backup_time = required_integer_param(&param, "backup-time")?;
 
         let backup_dir = BackupDir::new(backup_type, backup_id, backup_time)?;
 
@@ -1101,7 +1171,7 @@ pub const API_METHOD_DOWNLOAD_FILE_DECODED: ApiMethod = ApiMethod::new(
     true)
 );
 
-fn download_file_decoded(
+pub fn download_file_decoded(
     _parts: Parts,
     _req_body: Body,
     param: Value,
@@ -1110,16 +1180,16 @@ fn download_file_decoded(
 ) -> ApiResponseFuture {
 
     async move {
-        let store = tools::required_string_param(&param, "store")?;
+        let store = required_string_param(&param, "store")?;
         let datastore = DataStore::lookup_datastore(store)?;
 
         let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
 
-        let file_name = tools::required_string_param(&param, "file-name")?.to_owned();
+        let file_name = required_string_param(&param, "file-name")?.to_owned();
 
-        let backup_type = tools::required_string_param(&param, "backup-type")?;
-        let backup_id = tools::required_string_param(&param, "backup-id")?;
-        let backup_time = tools::required_integer_param(&param, "backup-time")?;
+        let backup_type = required_string_param(&param, "backup-type")?;
+        let backup_id = required_string_param(&param, "backup-id")?;
+        let backup_time = required_integer_param(&param, "backup-time")?;
 
         let backup_dir = BackupDir::new(backup_type, backup_id, backup_time)?;
 
@@ -1148,7 +1218,7 @@ fn download_file_decoded(
                 manifest.verify_file(&file_name, &csum, size)?;
 
                 let chunk_reader = LocalChunkReader::new(datastore, None, CryptMode::None);
-                let reader = AsyncIndexReader::new(index, chunk_reader);
+                let reader = CachedChunkReader::new(chunk_reader, index, 1).seekable();
                 Body::wrap_stream(AsyncReaderStream::new(reader)
                     .map_err(move |err| {
                         eprintln!("error during streaming of '{:?}' - {}", path, err);
@@ -1163,7 +1233,7 @@ fn download_file_decoded(
                 manifest.verify_file(&file_name, &csum, size)?;
 
                 let chunk_reader = LocalChunkReader::new(datastore, None, CryptMode::None);
-                let reader = AsyncIndexReader::new(index, chunk_reader);
+                let reader = CachedChunkReader::new(chunk_reader, index, 1).seekable();
                 Body::wrap_stream(AsyncReaderStream::with_buffer_size(reader, 4*1024*1024)
                     .map_err(move |err| {
                         eprintln!("error during streaming of '{:?}' - {}", path, err);
@@ -1215,7 +1285,7 @@ pub const API_METHOD_UPLOAD_BACKUP_LOG: ApiMethod = ApiMethod::new(
     &Permission::Privilege(&["datastore", "{store}"], PRIV_DATASTORE_BACKUP, false)
 );
 
-fn upload_backup_log(
+pub fn upload_backup_log(
     _parts: Parts,
     req_body: Body,
     param: Value,
@@ -1224,14 +1294,14 @@ fn upload_backup_log(
 ) -> ApiResponseFuture {
 
     async move {
-        let store = tools::required_string_param(&param, "store")?;
+        let store = required_string_param(&param, "store")?;
         let datastore = DataStore::lookup_datastore(store)?;
 
         let file_name =  CLIENT_LOG_BLOB_NAME;
 
-        let backup_type = tools::required_string_param(&param, "backup-type")?;
-        let backup_id = tools::required_string_param(&param, "backup-id")?;
-        let backup_time = tools::required_integer_param(&param, "backup-time")?;
+        let backup_type = required_string_param(&param, "backup-type")?;
+        let backup_id = required_string_param(&param, "backup-id")?;
+        let backup_time = required_integer_param(&param, "backup-time")?;
 
         let backup_dir = BackupDir::new(backup_type, backup_id, backup_time)?;
 
@@ -1294,16 +1364,14 @@ fn upload_backup_log(
     },
 )]
 /// Get the entries of the given path of the catalog
-fn catalog(
+pub fn catalog(
     store: String,
     backup_type: String,
     backup_id: String,
     backup_time: i64,
     filepath: String,
-    _param: Value,
-    _info: &ApiMethod,
     rpcenv: &mut dyn RpcEnvironment,
-) -> Result<Value, Error> {
+) -> Result<Vec<ArchiveEntry>, Error> {
     let datastore = DataStore::lookup_datastore(&store)?;
 
     let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
@@ -1335,112 +1403,14 @@ fn catalog(
     let reader = BufferedDynamicReader::new(index, chunk_reader);
 
     let mut catalog_reader = CatalogReader::new(reader);
-    let mut current = catalog_reader.root()?;
-    let mut components = vec![];
-
-
-    if filepath != "root" {
-        components = base64::decode(filepath)?;
-        if !components.is_empty() && components[0] == b'/' {
-            components.remove(0);
-        }
-        for component in components.split(|c| *c == b'/') {
-            if let Some(entry) = catalog_reader.lookup(&current, component)? {
-                current = entry;
-            } else {
-                bail!("path {:?} not found in catalog", &String::from_utf8_lossy(&components));
-            }
-        }
-    }
-
-    let mut res = Vec::new();
-
-    for direntry in catalog_reader.read_dir(&current)? {
-        let mut components = components.clone();
-        components.push(b'/');
-        components.extend(&direntry.name);
-        let path = base64::encode(components);
-        let text = String::from_utf8_lossy(&direntry.name);
-        let mut entry = json!({
-            "filepath": path,
-            "text": text,
-            "type": CatalogEntryType::from(&direntry.attr).to_string(),
-            "leaf": true,
-        });
-        match direntry.attr {
-            DirEntryAttribute::Directory { start: _ } => {
-                entry["leaf"] = false.into();
-            },
-            DirEntryAttribute::File { size, mtime } => {
-                entry["size"] = size.into();
-                entry["mtime"] = mtime.into();
-            },
-            _ => {},
-        }
-        res.push(entry);
-    }
 
-    Ok(res.into())
-}
-
-fn recurse_files<'a, T, W>(
-    zip: &'a mut ZipEncoder<W>,
-    decoder: &'a mut Accessor<T>,
-    prefix: &'a Path,
-    file: FileEntry<T>,
-) -> Pin<Box<dyn Future<Output = Result<(), Error>> + Send + 'a>>
-where
-    T: Clone + pxar::accessor::ReadAt + Unpin + Send + Sync + 'static,
-    W: tokio::io::AsyncWrite + Unpin + Send + 'static,
-{
-    Box::pin(async move {
-        let metadata = file.entry().metadata();
-        let path = file.entry().path().strip_prefix(&prefix)?.to_path_buf();
-
-        match file.kind() {
-            EntryKind::File { .. } => {
-                let entry = ZipEntry::new(
-                    path,
-                    metadata.stat.mtime.secs,
-                    metadata.stat.mode as u16,
-                    true,
-                );
-                zip.add_entry(entry, Some(file.contents().await?))
-                   .await
-                   .map_err(|err| format_err!("could not send file entry: {}", err))?;
-            }
-            EntryKind::Hardlink(_) => {
-                let realfile = decoder.follow_hardlink(&file).await?;
-                let entry = ZipEntry::new(
-                    path,
-                    metadata.stat.mtime.secs,
-                    metadata.stat.mode as u16,
-                    true,
-                );
-                zip.add_entry(entry, Some(realfile.contents().await?))
-                   .await
-                   .map_err(|err| format_err!("could not send file entry: {}", err))?;
-            }
-            EntryKind::Directory => {
-                let dir = file.enter_directory().await?;
-                let mut readdir = dir.read_dir();
-                let entry = ZipEntry::new(
-                    path,
-                    metadata.stat.mtime.secs,
-                    metadata.stat.mode as u16,
-                    false,
-                );
-                zip.add_entry::<FileContents<T>>(entry, None).await?;
-                while let Some(entry) = readdir.next().await {
-                    let entry = entry?.decode_entry().await?;
-                    recurse_files(zip, decoder, prefix, entry).await?;
-                }
-            }
-            _ => {} // ignore all else
-        };
+    let path = if filepath != "root" && filepath != "/" {
+        base64::decode(filepath)?
+    } else {
+        vec![b'/']
+    };
 
-        Ok(())
-    })
+    helpers::list_dir_content(&mut catalog_reader, &path)
 }
 
 #[sortable]
@@ -1462,7 +1432,7 @@ pub const API_METHOD_PXAR_FILE_DOWNLOAD: ApiMethod = ApiMethod::new(
     true)
 );
 
-fn pxar_file_download(
+pub fn pxar_file_download(
     _parts: Parts,
     _req_body: Body,
     param: Value,
@@ -1471,16 +1441,16 @@ fn pxar_file_download(
 ) -> ApiResponseFuture {
 
     async move {
-        let store = tools::required_string_param(&param, "store")?;
+        let store = required_string_param(&param, "store")?;
         let datastore = DataStore::lookup_datastore(&store)?;
 
         let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
 
-        let filepath = tools::required_string_param(&param, "filepath")?.to_owned();
+        let filepath = required_string_param(&param, "filepath")?.to_owned();
 
-        let backup_type = tools::required_string_param(&param, "backup-type")?;
-        let backup_id = tools::required_string_param(&param, "backup-id")?;
-        let backup_time = tools::required_integer_param(&param, "backup-time")?;
+        let backup_type = required_string_param(&param, "backup-type")?;
+        let backup_id = required_string_param(&param, "backup-id")?;
+        let backup_time = required_integer_param(&param, "backup-time")?;
 
         let backup_dir = BackupDir::new(backup_type, backup_id, backup_time)?;
 
@@ -1493,7 +1463,7 @@ fn pxar_file_download(
 
         let mut split = components.splitn(2, |c| *c == b'/');
         let pxar_name = std::str::from_utf8(split.next().unwrap())?;
-        let file_path = split.next().ok_or(format_err!("filepath looks strange '{}'", filepath))?;
+        let file_path = split.next().unwrap_or(b"/");
         let (manifest, files) = read_backup_index(&datastore, &backup_dir)?;
         for file in files {
             if file.filename == pxar_name && file.crypt_mode == Some(CryptMode::Encrypt) {
@@ -1518,9 +1488,10 @@ fn pxar_file_download(
 
         let decoder = Accessor::new(reader, archive_size).await?;
         let root = decoder.open_root().await?;
+        let path = OsStr::from_bytes(file_path).to_os_string();
         let file = root
-            .lookup(OsStr::from_bytes(file_path)).await?
-            .ok_or(format_err!("error opening '{:?}'", file_path))?;
+            .lookup(&path).await?
+            .ok_or_else(|| format_err!("error opening '{:?}'", path))?;
 
         let body = match file.kind() {
             EntryKind::File { .. } => Body::wrap_stream(
@@ -1534,37 +1505,19 @@ fn pxar_file_download(
                     .map_err(move |err| {
                         eprintln!(
                             "error during streaming of hardlink '{:?}' - {}",
-                            filepath, err
+                            path, err
                         );
                         err
                     }),
             ),
             EntryKind::Directory => {
                 let (sender, receiver) = tokio::sync::mpsc::channel(100);
-                let mut prefix = PathBuf::new();
-                let mut components = file.entry().path().components();
-                components.next_back(); // discar last
-                for comp in components {
-                    prefix.push(comp);
-                }
-
                 let channelwriter = AsyncChannelWriter::new(sender, 1024 * 1024);
-
-                crate::server::spawn_internal_task(async move {
-                    let mut zipencoder = ZipEncoder::new(channelwriter);
-                    let mut decoder = decoder;
-                    recurse_files(&mut zipencoder, &mut decoder, &prefix, file)
-                        .await
-                        .map_err(|err| eprintln!("error during creating of zip: {}", err))?;
-
-                    zipencoder
-                        .finish()
-                        .await
-                        .map_err(|err| eprintln!("error during finishing of zip: {}", err))
-                });
-
+                crate::server::spawn_internal_task(
+                    create_zip(channelwriter, decoder, path.clone(), false)
+                );
                 Body::wrap_stream(ReceiverStream::new(receiver).map_err(move |err| {
-                    eprintln!("error during streaming of zip '{:?}' - {}", filepath, err);
+                    eprintln!("error during streaming of zip '{:?}' - {}", path, err);
                     err
                 }))
             }
@@ -1599,7 +1552,7 @@ fn pxar_file_download(
     },
 )]
 /// Read datastore stats
-fn get_rrd_stats(
+pub fn get_rrd_stats(
     store: String,
     timeframe: RRDTimeFrameResolution,
     cf: RRDMode,
@@ -1619,6 +1572,86 @@ fn get_rrd_stats(
     )
 }
 
+#[api(
+    input: {
+        properties: {
+            store: {
+                schema: DATASTORE_SCHEMA,
+            },
+            "backup-type": {
+                schema: BACKUP_TYPE_SCHEMA,
+            },
+            "backup-id": {
+                schema: BACKUP_ID_SCHEMA,
+            },
+        },
+    },
+    access: {
+        permission: &Permission::Privilege(&["datastore", "{store}"], PRIV_DATASTORE_AUDIT | PRIV_DATASTORE_BACKUP, true),
+    },
+)]
+/// Get "notes" for a backup group
+pub fn get_group_notes(
+    store: String,
+    backup_type: String,
+    backup_id: String,
+    rpcenv: &mut dyn RpcEnvironment,
+) -> Result<String, Error> {
+    let datastore = DataStore::lookup_datastore(&store)?;
+
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+    let backup_group = BackupGroup::new(backup_type, backup_id);
+
+    check_priv_or_backup_owner(&datastore, &backup_group, &auth_id, PRIV_DATASTORE_AUDIT)?;
+
+    let note_path = get_group_note_path(&datastore, &backup_group);
+    Ok(file_read_optional_string(note_path)?.unwrap_or_else(|| "".to_owned()))
+}
+
+#[api(
+    input: {
+        properties: {
+            store: {
+                schema: DATASTORE_SCHEMA,
+            },
+            "backup-type": {
+                schema: BACKUP_TYPE_SCHEMA,
+            },
+            "backup-id": {
+                schema: BACKUP_ID_SCHEMA,
+            },
+            notes: {
+                description: "A multiline text.",
+            },
+        },
+    },
+    access: {
+        permission: &Permission::Privilege(&["datastore", "{store}"],
+                                           PRIV_DATASTORE_MODIFY | PRIV_DATASTORE_BACKUP,
+                                           true),
+    },
+)]
+/// Set "notes" for a backup group
+pub fn set_group_notes(
+    store: String,
+    backup_type: String,
+    backup_id: String,
+    notes: String,
+    rpcenv: &mut dyn RpcEnvironment,
+) -> Result<(), Error> {
+    let datastore = DataStore::lookup_datastore(&store)?;
+
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+    let backup_group = BackupGroup::new(backup_type, backup_id);
+
+    check_priv_or_backup_owner(&datastore, &backup_group, &auth_id, PRIV_DATASTORE_MODIFY)?;
+
+    let note_path = get_group_note_path(&datastore, &backup_group);
+    replace_file(note_path, notes.as_bytes(), CreateOptions::new())?;
+
+    Ok(())
+}
+
 #[api(
     input: {
         properties: {
@@ -1641,7 +1674,7 @@ fn get_rrd_stats(
     },
 )]
 /// Get "notes" for a specific backup
-fn get_notes(
+pub fn get_notes(
     store: String,
     backup_type: String,
     backup_id: String,
@@ -1691,7 +1724,7 @@ fn get_notes(
     },
 )]
 /// Set "notes" for a specific backup
-fn set_notes(
+pub fn set_notes(
     store: String,
     backup_type: String,
     backup_id: String,
@@ -1736,7 +1769,7 @@ fn set_notes(
     },
 )]
 /// Change owner of a backup group
-fn set_backup_owner(
+pub fn set_backup_owner(
     store: String,
     backup_type: String,
     backup_id: String,
@@ -1843,10 +1876,17 @@ const DATASTORE_INFO_SUBDIRS: SubdirMap = &[
             .get(&API_METHOD_GARBAGE_COLLECTION_STATUS)
             .post(&API_METHOD_START_GARBAGE_COLLECTION)
     ),
+    (
+        "group-notes",
+        &Router::new()
+            .get(&API_METHOD_GET_GROUP_NOTES)
+            .put(&API_METHOD_SET_GROUP_NOTES)
+    ),
     (
         "groups",
         &Router::new()
             .get(&API_METHOD_LIST_GROUPS)
+            .delete(&API_METHOD_DELETE_GROUP)
     ),
     (
         "notes",
@@ -1859,6 +1899,11 @@ const DATASTORE_INFO_SUBDIRS: SubdirMap = &[
         &Router::new()
             .post(&API_METHOD_PRUNE)
     ),
+    (
+        "prune-datastore",
+        &Router::new()
+            .post(&API_METHOD_PRUNE_DATASTORE)
+    ),
     (
         "pxar-file-download",
         &Router::new()