From 2b7f8dd5ea8c81198a76aa2b275e446c6ba7cb4b Mon Sep 17 00:00:00 2001 From: Wolfgang Bumiller Date: Mon, 19 Jul 2021 10:50:18 +0200 Subject: [PATCH] move client to pbs-client subcrate Signed-off-by: Wolfgang Bumiller --- Cargo.toml | 2 + Makefile | 1 + examples/download-speed.rs | 4 +- examples/upload-speed.rs | 4 +- pbs-api-types/src/lib.rs | 55 +++-- pbs-api-types/src/user.rs | 205 ++++++++++++++++++ pbs-api-types/src/userid.rs | 1 - pbs-client/Cargo.toml | 40 ++++ .../src}/backup_reader.rs | 2 +- {src/client => pbs-client/src}/backup_repo.rs | 0 .../src}/backup_specification.rs | 0 .../src}/backup_writer.rs | 2 +- .../src}/catalog_shell.rs | 18 +- {src/client => pbs-client/src}/http_client.rs | 0 src/client/mod.rs => pbs-client/src/lib.rs | 4 + .../src}/merge_known_chunks.rs | 0 .../src}/pipe_to_stream.rs | 0 {src => pbs-client/src}/pxar/create.rs | 7 +- {src => pbs-client/src}/pxar/dir_stack.rs | 0 {src => pbs-client/src}/pxar/extract.rs | 6 +- {src => pbs-client/src}/pxar/flags.rs | 0 {src => pbs-client/src}/pxar/fuse.rs | 2 +- {src => pbs-client/src}/pxar/metadata.rs | 3 +- {src => pbs-client/src}/pxar/mod.rs | 2 +- {src => pbs-client/src}/pxar/tools.rs | 0 .../src}/pxar_backup_stream.rs | 0 .../src}/remote_chunk_reader.rs | 0 {src/client => pbs-client/src}/task_log.rs | 0 .../src/tools}/key_source.rs | 2 +- .../src/tools}/mod.rs | 21 +- .../client => pbs-client/src}/vsock_client.rs | 0 pbs-tools/Cargo.toml | 9 +- {src/tools => pbs-tools/src}/acl.rs | 0 pbs-tools/src/compression.rs | 194 +++++++++++++++++ pbs-tools/src/fs.rs | 59 ++++- pbs-tools/src/lib.rs | 5 + pbs-tools/src/ops.rs | 12 + pbs-tools/src/str.rs | 10 + {src/tools => pbs-tools/src}/xattr.rs | 0 {src/tools => pbs-tools/src}/zip.rs | 2 +- src/api2/access/user.rs | 91 ++------ src/api2/admin/datastore.rs | 3 +- src/api2/backup/mod.rs | 1 + src/api2/config/remote.rs | 3 +- src/api2/pull.rs | 3 +- src/api2/reader/mod.rs | 1 + src/api2/types/mod.rs | 8 - src/backup/mod.rs | 18 -- src/bin/dump-catalog-shell-cli.rs | 2 +- src/bin/proxmox-backup-client.rs | 85 ++++---- src/bin/proxmox-backup-manager.rs | 2 +- src/bin/proxmox-file-restore.rs | 37 ++-- src/bin/proxmox-restore-daemon.rs | 3 +- src/bin/proxmox-tape.rs | 5 +- src/bin/proxmox_backup_client/benchmark.rs | 7 +- src/bin/proxmox_backup_client/catalog.rs | 11 +- src/bin/proxmox_backup_client/key.rs | 29 ++- src/bin/proxmox_backup_client/mount.rs | 15 +- src/bin/proxmox_backup_client/snapshot.rs | 8 +- src/bin/proxmox_backup_client/task.rs | 2 +- src/bin/proxmox_backup_manager/datastore.rs | 6 +- src/bin/proxmox_file_restore/block_driver.rs | 15 +- .../proxmox_file_restore/block_driver_qemu.rs | 14 +- src/bin/proxmox_file_restore/qemu_helper.rs | 3 +- src/bin/proxmox_restore_daemon/api.rs | 5 +- src/bin/proxmox_tape/backup_job.rs | 6 +- src/bin/pxar.rs | 27 +-- src/config/user.rs | 150 +------------ src/lib.rs | 4 - src/server/pull.rs | 2 +- src/server/rest.rs | 4 +- src/tools/compression.rs | 193 ----------------- src/tools/mod.rs | 87 +------- tests/catar.rs | 5 +- 74 files changed, 788 insertions(+), 739 deletions(-) create mode 100644 pbs-api-types/src/user.rs create mode 100644 pbs-client/Cargo.toml rename {src/client => pbs-client/src}/backup_reader.rs (98%) rename {src/client => pbs-client/src}/backup_repo.rs (100%) rename {src/client => pbs-client/src}/backup_specification.rs (100%) rename {src/client => pbs-client/src}/backup_writer.rs (99%) rename {src/backup => pbs-client/src}/catalog_shell.rs (98%) rename {src/client => pbs-client/src}/http_client.rs (100%) rename src/client/mod.rs => pbs-client/src/lib.rs (96%) rename {src/client => pbs-client/src}/merge_known_chunks.rs (100%) rename {src/client => pbs-client/src}/pipe_to_stream.rs (100%) rename {src => pbs-client/src}/pxar/create.rs (99%) rename {src => pbs-client/src}/pxar/dir_stack.rs (100%) rename {src => pbs-client/src}/pxar/extract.rs (99%) rename {src => pbs-client/src}/pxar/flags.rs (100%) rename {src => pbs-client/src}/pxar/fuse.rs (99%) rename {src => pbs-client/src}/pxar/metadata.rs (99%) rename {src => pbs-client/src}/pxar/mod.rs (100%) rename {src => pbs-client/src}/pxar/tools.rs (100%) rename {src/client => pbs-client/src}/pxar_backup_stream.rs (100%) rename {src/client => pbs-client/src}/remote_chunk_reader.rs (100%) rename {src/client => pbs-client/src}/task_log.rs (100%) rename {src/bin/proxmox_client_tools => pbs-client/src/tools}/key_source.rs (99%) rename {src/bin/proxmox_client_tools => pbs-client/src/tools}/mod.rs (93%) rename {src/client => pbs-client/src}/vsock_client.rs (100%) rename {src/tools => pbs-tools/src}/acl.rs (100%) create mode 100644 pbs-tools/src/compression.rs create mode 100644 pbs-tools/src/ops.rs rename {src/tools => pbs-tools/src}/xattr.rs (100%) rename {src/tools => pbs-tools/src}/zip.rs (99%) diff --git a/Cargo.toml b/Cargo.toml index 9c41605a..8dcd50d1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,7 @@ exclude = [ "build", "debian", "tests/catar_data/test_symlink/symlink1"] [workspace] members = [ "pbs-buildcfg", + "pbs-client", "pbs-datastore", "pbs-runtime", "pbs-systemd", @@ -95,6 +96,7 @@ proxmox-openid = "0.6.0" pbs-api-types = { path = "pbs-api-types" } pbs-buildcfg = { path = "pbs-buildcfg" } +pbs-client = { path = "pbs-client" } pbs-datastore = { path = "pbs-datastore" } pbs-runtime = { path = "pbs-runtime" } pbs-systemd = { path = "pbs-systemd" } diff --git a/Makefile b/Makefile index 0ca0457e..e0fcaf62 100644 --- a/Makefile +++ b/Makefile @@ -33,6 +33,7 @@ RESTORE_BIN := \ SUBCRATES := \ pbs-api-types \ pbs-buildcfg \ + pbs-client \ pbs-datastore \ pbs-runtime \ pbs-systemd \ diff --git a/examples/download-speed.rs b/examples/download-speed.rs index 90b007e0..471d30f0 100644 --- a/examples/download-speed.rs +++ b/examples/download-speed.rs @@ -2,8 +2,8 @@ use std::io::Write; use anyhow::{Error}; -use proxmox_backup::api2::types::Authid; -use proxmox_backup::client::{HttpClient, HttpClientOptions, BackupReader}; +use pbs_api_types::Authid; +use pbs_client::{HttpClient, HttpClientOptions, BackupReader}; pub struct DummyWriter { bytes: usize, diff --git a/examples/upload-speed.rs b/examples/upload-speed.rs index c2c549ca..7c2fd0e7 100644 --- a/examples/upload-speed.rs +++ b/examples/upload-speed.rs @@ -1,7 +1,7 @@ use anyhow::{Error}; -use proxmox_backup::api2::types::Authid; -use proxmox_backup::client::*; +use pbs_client::{HttpClient, HttpClientOptions, BackupWriter}; +use pbs_api_types::Authid; async fn upload_speed() -> Result { diff --git a/pbs-api-types/src/lib.rs b/pbs-api-types/src/lib.rs index c07699d1..00d1e3a5 100644 --- a/pbs-api-types/src/lib.rs +++ b/pbs-api-types/src/lib.rs @@ -40,6 +40,13 @@ pub use userid::{Tokenname, TokennameRef}; pub use userid::{Username, UsernameRef}; pub use userid::{PROXMOX_GROUP_ID_SCHEMA, PROXMOX_TOKEN_ID_SCHEMA, PROXMOX_TOKEN_NAME_SCHEMA}; +#[macro_use] +mod user; +pub use user::{ApiToken, User, UserWithTokens}; +pub use user::{ + EMAIL_SCHEMA, ENABLE_USER_SCHEMA, EXPIRE_USER_SCHEMA, FIRST_NAME_SCHEMA, LAST_NAME_SCHEMA, +}; + pub mod upid; pub use upid::UPID; @@ -146,35 +153,33 @@ pub const CERT_FINGERPRINT_SHA256_SCHEMA: Schema = .format(&FINGERPRINT_SHA256_FORMAT) .schema(); -pub const PRUNE_SCHEMA_KEEP_DAILY: Schema = IntegerSchema::new( - "Number of daily backups to keep.") +pub const PRUNE_SCHEMA_KEEP_DAILY: Schema = IntegerSchema::new("Number of daily backups to keep.") .minimum(1) .schema(); -pub const PRUNE_SCHEMA_KEEP_HOURLY: Schema = IntegerSchema::new( - "Number of hourly backups to keep.") - .minimum(1) - .schema(); +pub const PRUNE_SCHEMA_KEEP_HOURLY: Schema = + IntegerSchema::new("Number of hourly backups to keep.") + .minimum(1) + .schema(); -pub const PRUNE_SCHEMA_KEEP_LAST: Schema = IntegerSchema::new( - "Number of backups to keep.") +pub const PRUNE_SCHEMA_KEEP_LAST: Schema = IntegerSchema::new("Number of backups to keep.") .minimum(1) .schema(); -pub const PRUNE_SCHEMA_KEEP_MONTHLY: Schema = IntegerSchema::new( - "Number of monthly backups to keep.") - .minimum(1) - .schema(); +pub const PRUNE_SCHEMA_KEEP_MONTHLY: Schema = + IntegerSchema::new("Number of monthly backups to keep.") + .minimum(1) + .schema(); -pub const PRUNE_SCHEMA_KEEP_WEEKLY: Schema = IntegerSchema::new( - "Number of weekly backups to keep.") - .minimum(1) - .schema(); +pub const PRUNE_SCHEMA_KEEP_WEEKLY: Schema = + IntegerSchema::new("Number of weekly backups to keep.") + .minimum(1) + .schema(); -pub const PRUNE_SCHEMA_KEEP_YEARLY: Schema = IntegerSchema::new( - "Number of yearly backups to keep.") - .minimum(1) - .schema(); +pub const PRUNE_SCHEMA_KEEP_YEARLY: Schema = + IntegerSchema::new("Number of yearly backups to keep.") + .minimum(1) + .schema(); pub const PROXMOX_SAFE_ID_FORMAT: ApiStringFormat = ApiStringFormat::Pattern(&PROXMOX_SAFE_ID_REGEX); @@ -186,6 +191,14 @@ pub const SINGLE_LINE_COMMENT_SCHEMA: Schema = StringSchema::new("Comment (singl .format(&SINGLE_LINE_COMMENT_FORMAT) .schema(); +pub const PROXMOX_CONFIG_DIGEST_SCHEMA: Schema = StringSchema::new( + "Prevent changes if current configuration file has different \ + SHA256 digest. This can be used to prevent concurrent \ + modifications.", +) +.format(&PVE_CONFIG_DIGEST_FORMAT) +.schema(); + pub const BACKUP_ID_FORMAT: ApiStringFormat = ApiStringFormat::Pattern(&BACKUP_ID_REGEX); /// API schema format definition for repository URLs @@ -411,7 +424,7 @@ pub struct GroupListItem { #[serde(skip_serializing_if = "Option::is_none")] pub owner: Option, /// The first line from group "notes" - #[serde(skip_serializing_if="Option::is_none")] + #[serde(skip_serializing_if = "Option::is_none")] pub comment: Option, } diff --git a/pbs-api-types/src/user.rs b/pbs-api-types/src/user.rs new file mode 100644 index 00000000..9111ccea --- /dev/null +++ b/pbs-api-types/src/user.rs @@ -0,0 +1,205 @@ +use serde::{Deserialize, Serialize}; + +use proxmox::api::api; +use proxmox::api::schema::{BooleanSchema, IntegerSchema, Schema, StringSchema}; + +use super::{SINGLE_LINE_COMMENT_FORMAT, SINGLE_LINE_COMMENT_SCHEMA}; +use super::userid::{Authid, Userid, PROXMOX_TOKEN_ID_SCHEMA}; + +pub const ENABLE_USER_SCHEMA: Schema = BooleanSchema::new( + "Enable the account (default). You can set this to '0' to disable the account.") + .default(true) + .schema(); + +pub const EXPIRE_USER_SCHEMA: Schema = IntegerSchema::new( + "Account expiration date (seconds since epoch). '0' means no expiration date.") + .default(0) + .minimum(0) + .schema(); + +pub const FIRST_NAME_SCHEMA: Schema = StringSchema::new("First name.") + .format(&SINGLE_LINE_COMMENT_FORMAT) + .min_length(2) + .max_length(64) + .schema(); + +pub const LAST_NAME_SCHEMA: Schema = StringSchema::new("Last name.") + .format(&SINGLE_LINE_COMMENT_FORMAT) + .min_length(2) + .max_length(64) + .schema(); + +pub const EMAIL_SCHEMA: Schema = StringSchema::new("E-Mail Address.") + .format(&SINGLE_LINE_COMMENT_FORMAT) + .min_length(2) + .max_length(64) + .schema(); + +#[api( + properties: { + userid: { + type: Userid, + }, + comment: { + optional: true, + schema: SINGLE_LINE_COMMENT_SCHEMA, + }, + enable: { + optional: true, + schema: ENABLE_USER_SCHEMA, + }, + expire: { + optional: true, + schema: EXPIRE_USER_SCHEMA, + }, + firstname: { + optional: true, + schema: FIRST_NAME_SCHEMA, + }, + lastname: { + schema: LAST_NAME_SCHEMA, + optional: true, + }, + email: { + schema: EMAIL_SCHEMA, + optional: true, + }, + tokens: { + type: Array, + optional: true, + description: "List of user's API tokens.", + items: { + type: ApiToken + }, + }, + } +)] +#[derive(Serialize,Deserialize)] +/// User properties with added list of ApiTokens +pub struct UserWithTokens { + pub userid: Userid, + #[serde(skip_serializing_if="Option::is_none")] + pub comment: Option, + #[serde(skip_serializing_if="Option::is_none")] + pub enable: Option, + #[serde(skip_serializing_if="Option::is_none")] + pub expire: Option, + #[serde(skip_serializing_if="Option::is_none")] + pub firstname: Option, + #[serde(skip_serializing_if="Option::is_none")] + pub lastname: Option, + #[serde(skip_serializing_if="Option::is_none")] + pub email: Option, + #[serde(skip_serializing_if="Vec::is_empty", default)] + pub tokens: Vec, +} + +#[api( + properties: { + tokenid: { + schema: PROXMOX_TOKEN_ID_SCHEMA, + }, + comment: { + optional: true, + schema: SINGLE_LINE_COMMENT_SCHEMA, + }, + enable: { + optional: true, + schema: ENABLE_USER_SCHEMA, + }, + expire: { + optional: true, + schema: EXPIRE_USER_SCHEMA, + }, + } +)] +#[derive(Serialize,Deserialize)] +/// ApiToken properties. +pub struct ApiToken { + pub tokenid: Authid, + #[serde(skip_serializing_if="Option::is_none")] + pub comment: Option, + #[serde(skip_serializing_if="Option::is_none")] + pub enable: Option, + #[serde(skip_serializing_if="Option::is_none")] + pub expire: Option, +} + +impl ApiToken { + pub fn is_active(&self) -> bool { + if !self.enable.unwrap_or(true) { + return false; + } + if let Some(expire) = self.expire { + let now = proxmox::tools::time::epoch_i64(); + if expire > 0 && expire <= now { + return false; + } + } + true + } +} + +#[api( + properties: { + userid: { + type: Userid, + }, + comment: { + optional: true, + schema: SINGLE_LINE_COMMENT_SCHEMA, + }, + enable: { + optional: true, + schema: ENABLE_USER_SCHEMA, + }, + expire: { + optional: true, + schema: EXPIRE_USER_SCHEMA, + }, + firstname: { + optional: true, + schema: FIRST_NAME_SCHEMA, + }, + lastname: { + schema: LAST_NAME_SCHEMA, + optional: true, + }, + email: { + schema: EMAIL_SCHEMA, + optional: true, + }, + } +)] +#[derive(Serialize,Deserialize)] +/// User properties. +pub struct User { + pub userid: Userid, + #[serde(skip_serializing_if="Option::is_none")] + pub comment: Option, + #[serde(skip_serializing_if="Option::is_none")] + pub enable: Option, + #[serde(skip_serializing_if="Option::is_none")] + pub expire: Option, + #[serde(skip_serializing_if="Option::is_none")] + pub firstname: Option, + #[serde(skip_serializing_if="Option::is_none")] + pub lastname: Option, + #[serde(skip_serializing_if="Option::is_none")] + pub email: Option, +} + +impl User { + pub fn is_active(&self) -> bool { + if !self.enable.unwrap_or(true) { + return false; + } + if let Some(expire) = self.expire { + let now = proxmox::tools::time::epoch_i64(); + if expire > 0 && expire <= now { + return false; + } + } + true + } +} diff --git a/pbs-api-types/src/userid.rs b/pbs-api-types/src/userid.rs index 08335b93..e931181e 100644 --- a/pbs-api-types/src/userid.rs +++ b/pbs-api-types/src/userid.rs @@ -98,7 +98,6 @@ pub const PROXMOX_AUTH_REALM_STRING_SCHEMA: StringSchema = .max_length(32); pub const PROXMOX_AUTH_REALM_SCHEMA: Schema = PROXMOX_AUTH_REALM_STRING_SCHEMA.schema(); - #[api( type: String, format: &PROXMOX_USER_NAME_FORMAT, diff --git a/pbs-client/Cargo.toml b/pbs-client/Cargo.toml new file mode 100644 index 00000000..c5dbf149 --- /dev/null +++ b/pbs-client/Cargo.toml @@ -0,0 +1,40 @@ +[package] +name = "pbs-client" +version = "0.1.0" +authors = ["Wolfgang Bumiller "] +edition = "2018" +description = "The main proxmox backup client crate" + +[dependencies] +anyhow = "1.0" +bitflags = "1.2.1" +bytes = "1.0" +futures = "0.3" +h2 = { version = "0.3", features = [ "stream" ] } +http = "0.2" +hyper = { version = "0.14", features = [ "full" ] } +lazy_static = "1.4" +libc = "0.2" +nix = "0.19.1" +openssl = "0.10" +percent-encoding = "2.1" +pin-project = "1.0" +regex = "1.2" +rustyline = "7" +serde_json = "1.0" +tokio = { version = "1.6", features = [ "fs", "signal" ] } +tokio-stream = "0.1.0" +tower-service = "0.3.0" +xdg = "2.2" + +pathpatterns = "0.1.2" +proxmox = { version = "0.11.5", default-features = false, features = [ "cli" ] } +proxmox-fuse = "0.1.1" +proxmox-http = { version = "0.2.1", features = [ "client", "http-helpers", "websocket" ] } +pxar = { version = "0.10.1", features = [ "tokio-io" ] } + +pbs-api-types = { path = "../pbs-api-types" } +pbs-buildcfg = { path = "../pbs-buildcfg" } +pbs-datastore = { path = "../pbs-datastore" } +pbs-runtime = { path = "../pbs-runtime" } +pbs-tools = { path = "../pbs-tools" } diff --git a/src/client/backup_reader.rs b/pbs-client/src/backup_reader.rs similarity index 98% rename from src/client/backup_reader.rs rename to pbs-client/src/backup_reader.rs index e04494d1..1702d441 100644 --- a/src/client/backup_reader.rs +++ b/pbs-client/src/backup_reader.rs @@ -9,7 +9,7 @@ use serde_json::{json, Value}; use proxmox::tools::digest_to_hex; -use pbs_datastore::{CryptConfig, BackupManifest}; +use pbs_datastore::{PROXMOX_BACKUP_READER_PROTOCOL_ID_V1, CryptConfig, BackupManifest}; use pbs_datastore::data_blob::DataBlob; use pbs_datastore::data_blob_reader::DataBlobReader; use pbs_datastore::dynamic_index::DynamicIndexReader; diff --git a/src/client/backup_repo.rs b/pbs-client/src/backup_repo.rs similarity index 100% rename from src/client/backup_repo.rs rename to pbs-client/src/backup_repo.rs diff --git a/src/client/backup_specification.rs b/pbs-client/src/backup_specification.rs similarity index 100% rename from src/client/backup_specification.rs rename to pbs-client/src/backup_specification.rs diff --git a/src/client/backup_writer.rs b/pbs-client/src/backup_writer.rs similarity index 99% rename from src/client/backup_writer.rs rename to pbs-client/src/backup_writer.rs index 8b3ddefa..5c15f27e 100644 --- a/src/client/backup_writer.rs +++ b/pbs-client/src/backup_writer.rs @@ -14,7 +14,7 @@ use tokio_stream::wrappers::ReceiverStream; use proxmox::tools::digest_to_hex; -use pbs_datastore::{CATALOG_NAME, CryptConfig}; +use pbs_datastore::{CATALOG_NAME, PROXMOX_BACKUP_PROTOCOL_ID_V1, CryptConfig}; use pbs_datastore::data_blob::{ChunkInfo, DataBlob, DataChunkBuilder}; use pbs_datastore::dynamic_index::DynamicIndexReader; use pbs_datastore::fixed_index::FixedIndexReader; diff --git a/src/backup/catalog_shell.rs b/pbs-client/src/catalog_shell.rs similarity index 98% rename from src/backup/catalog_shell.rs rename to pbs-client/src/catalog_shell.rs index b186ac6c..defa6238 100644 --- a/src/backup/catalog_shell.rs +++ b/pbs-client/src/catalog_shell.rs @@ -18,13 +18,14 @@ use proxmox::api::cli::{self, CliCommand, CliCommandMap, CliHelper, CommandLineI use proxmox::tools::fs::{create_path, CreateOptions}; use pxar::{EntryKind, Metadata}; -use crate::backup::catalog::{self, DirEntryAttribute}; -use crate::pxar::fuse::{Accessor, FileEntry}; -use crate::pxar::Flags; use pbs_runtime::block_in_place; -use crate::tools::ControlFlow; +use pbs_datastore::catalog::{self, DirEntryAttribute}; +use pbs_tools::ops::ControlFlow; + +use crate::pxar::Flags; +use crate::pxar::fuse::{Accessor, FileEntry}; -type CatalogReader = crate::backup::CatalogReader; +type CatalogReader = pbs_datastore::catalog::CatalogReader; const MAX_SYMLINK_COUNT: usize = 40; @@ -78,13 +79,13 @@ pub fn catalog_shell_cli() -> CommandLineInterface { "restore-selected", CliCommand::new(&API_METHOD_RESTORE_SELECTED_COMMAND) .arg_param(&["target"]) - .completion_cb("target", crate::tools::complete_file_name), + .completion_cb("target", pbs_tools::fs::complete_file_name), ) .insert( "restore", CliCommand::new(&API_METHOD_RESTORE_COMMAND) .arg_param(&["target"]) - .completion_cb("target", crate::tools::complete_file_name), + .completion_cb("target", pbs_tools::fs::complete_file_name), ) .insert( "find", @@ -985,7 +986,8 @@ impl Shell { .metadata() .clone(); - let extractor = crate::pxar::extract::Extractor::new(rootdir, root_meta, true, Flags::DEFAULT); + let extractor = + crate::pxar::extract::Extractor::new(rootdir, root_meta, true, Flags::DEFAULT); let mut extractor = ExtractorState::new( &mut self.catalog, diff --git a/src/client/http_client.rs b/pbs-client/src/http_client.rs similarity index 100% rename from src/client/http_client.rs rename to pbs-client/src/http_client.rs diff --git a/src/client/mod.rs b/pbs-client/src/lib.rs similarity index 96% rename from src/client/mod.rs rename to pbs-client/src/lib.rs index 8ff00018..78e9dad2 100644 --- a/src/client/mod.rs +++ b/pbs-client/src/lib.rs @@ -10,6 +10,10 @@ use pbs_tools::ticket::Ticket; use pbs_tools::cert::CertInfo; use pbs_tools::auth::private_auth_key; +pub mod catalog_shell; +pub mod pxar; +pub mod tools; + mod merge_known_chunks; pub mod pipe_to_stream; diff --git a/src/client/merge_known_chunks.rs b/pbs-client/src/merge_known_chunks.rs similarity index 100% rename from src/client/merge_known_chunks.rs rename to pbs-client/src/merge_known_chunks.rs diff --git a/src/client/pipe_to_stream.rs b/pbs-client/src/pipe_to_stream.rs similarity index 100% rename from src/client/pipe_to_stream.rs rename to pbs-client/src/pipe_to_stream.rs diff --git a/src/pxar/create.rs b/pbs-client/src/pxar/create.rs similarity index 99% rename from src/pxar/create.rs rename to pbs-client/src/pxar/create.rs index a3ce3565..96888c7c 100644 --- a/src/pxar/create.rs +++ b/pbs-client/src/pxar/create.rs @@ -23,14 +23,15 @@ use proxmox::c_str; use proxmox::sys::error::SysError; use proxmox::tools::fd::RawFdNum; use proxmox::tools::vec; +use proxmox::tools::fd::Fd; use pbs_datastore::catalog::BackupCatalogWriter; -use pbs_tools::fs; +use pbs_tools::{acl, fs, xattr}; +use pbs_tools::str::strip_ascii_whitespace; use crate::pxar::metadata::errno_is_unsupported; use crate::pxar::Flags; use crate::pxar::tools::assert_single_path_component; -use crate::tools::{acl, xattr, Fd}; /// Pxar options for creating a pxar archive/stream #[derive(Default, Clone)] @@ -362,7 +363,7 @@ impl Archiver { } }; - let line = crate::tools::strip_ascii_whitespace(&line); + let line = strip_ascii_whitespace(&line); if line.is_empty() || line[0] == b'#' { continue; diff --git a/src/pxar/dir_stack.rs b/pbs-client/src/pxar/dir_stack.rs similarity index 100% rename from src/pxar/dir_stack.rs rename to pbs-client/src/pxar/dir_stack.rs diff --git a/src/pxar/extract.rs b/pbs-client/src/pxar/extract.rs similarity index 99% rename from src/pxar/extract.rs rename to pbs-client/src/pxar/extract.rs index 16b2b499..a03b5a54 100644 --- a/src/pxar/extract.rs +++ b/pbs-client/src/pxar/extract.rs @@ -27,12 +27,12 @@ use proxmox::tools::{ io::{sparse_copy, sparse_copy_async}, }; +use pbs_tools::zip::{ZipEncoder, ZipEntry}; + use crate::pxar::dir_stack::PxarDirStack; use crate::pxar::metadata; use crate::pxar::Flags; -use crate::tools::zip::{ZipEncoder, ZipEntry}; - pub struct PxarExtractOptions<'a> { pub match_list: &'a[MatchEntry], pub extract_match_default: bool, @@ -215,7 +215,7 @@ where } /// Common state for file extraction. -pub(crate) struct Extractor { +pub struct Extractor { feature_flags: Flags, allow_existing_dirs: bool, dir_stack: PxarDirStack, diff --git a/src/pxar/flags.rs b/pbs-client/src/pxar/flags.rs similarity index 100% rename from src/pxar/flags.rs rename to pbs-client/src/pxar/flags.rs diff --git a/src/pxar/fuse.rs b/pbs-client/src/pxar/fuse.rs similarity index 99% rename from src/pxar/fuse.rs rename to pbs-client/src/pxar/fuse.rs index a5001cbe..1f5b1dd8 100644 --- a/src/pxar/fuse.rs +++ b/pbs-client/src/pxar/fuse.rs @@ -26,7 +26,7 @@ use pxar::accessor::{self, EntryRangeInfo, ReadAt}; use proxmox_fuse::requests::{self, FuseRequest}; use proxmox_fuse::{EntryParam, Fuse, ReplyBufState, Request, ROOT_ID}; -use crate::tools::xattr; +use pbs_tools::xattr; /// We mark inodes for regular files this way so we know how to access them. const NON_DIRECTORY_INODE: u64 = 1u64 << 63; diff --git a/src/pxar/metadata.rs b/pbs-client/src/pxar/metadata.rs similarity index 99% rename from src/pxar/metadata.rs rename to pbs-client/src/pxar/metadata.rs index e399c63c..2d27270a 100644 --- a/src/pxar/metadata.rs +++ b/pbs-client/src/pxar/metadata.rs @@ -13,11 +13,10 @@ use proxmox::c_result; use proxmox::sys::error::SysError; use proxmox::tools::fd::RawFdNum; -use pbs_tools::fs; +use pbs_tools::{acl, fs, xattr}; use crate::pxar::tools::perms_from_metadata; use crate::pxar::Flags; -use crate::tools::{acl, xattr}; // // utility functions diff --git a/src/pxar/mod.rs b/pbs-client/src/pxar/mod.rs similarity index 100% rename from src/pxar/mod.rs rename to pbs-client/src/pxar/mod.rs index 6ad913dc..f20a1f9e 100644 --- a/src/pxar/mod.rs +++ b/pbs-client/src/pxar/mod.rs @@ -50,8 +50,8 @@ pub(crate) mod create; pub(crate) mod dir_stack; pub(crate) mod extract; -pub(crate) mod metadata; pub mod fuse; +pub(crate) mod metadata; pub(crate) mod tools; mod flags; diff --git a/src/pxar/tools.rs b/pbs-client/src/pxar/tools.rs similarity index 100% rename from src/pxar/tools.rs rename to pbs-client/src/pxar/tools.rs diff --git a/src/client/pxar_backup_stream.rs b/pbs-client/src/pxar_backup_stream.rs similarity index 100% rename from src/client/pxar_backup_stream.rs rename to pbs-client/src/pxar_backup_stream.rs diff --git a/src/client/remote_chunk_reader.rs b/pbs-client/src/remote_chunk_reader.rs similarity index 100% rename from src/client/remote_chunk_reader.rs rename to pbs-client/src/remote_chunk_reader.rs diff --git a/src/client/task_log.rs b/pbs-client/src/task_log.rs similarity index 100% rename from src/client/task_log.rs rename to pbs-client/src/task_log.rs diff --git a/src/bin/proxmox_client_tools/key_source.rs b/pbs-client/src/tools/key_source.rs similarity index 99% rename from src/bin/proxmox_client_tools/key_source.rs rename to pbs-client/src/tools/key_source.rs index fee00723..340fe234 100644 --- a/src/bin/proxmox_client_tools/key_source.rs +++ b/pbs-client/src/tools/key_source.rs @@ -10,7 +10,7 @@ use proxmox::api::schema::*; use proxmox::sys::linux::tty; use proxmox::tools::fs::file_get_contents; -use proxmox_backup::backup::CryptMode; +use pbs_api_types::CryptMode; pub const DEFAULT_ENCRYPTION_KEY_FILE_NAME: &str = "encryption-key.json"; pub const DEFAULT_MASTER_PUBKEY_FILE_NAME: &str = "master-public.pem"; diff --git a/src/bin/proxmox_client_tools/mod.rs b/pbs-client/src/tools/mod.rs similarity index 93% rename from src/bin/proxmox_client_tools/mod.rs rename to pbs-client/src/tools/mod.rs index a54abe06..7b932b63 100644 --- a/src/bin/proxmox_client_tools/mod.rs +++ b/pbs-client/src/tools/mod.rs @@ -10,14 +10,11 @@ use proxmox::{ tools::fs::file_get_json, }; -use pbs_api_types::{BACKUP_REPO_URL, Authid}; -use pbs_buildcfg; +use pbs_api_types::{BACKUP_REPO_URL, Authid, UserWithTokens}; use pbs_datastore::BackupDir; use pbs_tools::json::json_object_to_query; -use proxmox_backup::api2::access::user::UserWithTokens; -use proxmox_backup::client::{BackupRepository, HttpClient, HttpClientOptions}; -use proxmox_backup::tools; +use crate::{BackupRepository, HttpClient, HttpClientOptions}; pub mod key_source; @@ -342,7 +339,7 @@ pub fn complete_backup_source(arg: &str, param: &HashMap) -> Vec return result; } - let files = tools::complete_file_name(data[1], param); + let files = pbs_tools::fs::complete_file_name(data[1], param); for file in files { result.push(format!("{}:{}", data[0], file)); @@ -375,15 +372,3 @@ pub fn place_xdg_file( .and_then(|base| base.place_config_file(file_name).map_err(Error::from)) .with_context(|| format!("failed to place {} in xdg home", description)) } - -/// Returns a runtime dir owned by the current user. -/// Note that XDG_RUNTIME_DIR is not always available, especially for non-login users like -/// "www-data", so we use a custom one in /run/proxmox-backup/ instead. -pub fn get_user_run_dir() -> Result { - let uid = nix::unistd::Uid::current(); - let mut path: std::path::PathBuf = pbs_buildcfg::PROXMOX_BACKUP_RUN_DIR.into(); - path.push(uid.to_string()); - tools::create_run_dir()?; - std::fs::create_dir_all(&path)?; - Ok(path) -} diff --git a/src/client/vsock_client.rs b/pbs-client/src/vsock_client.rs similarity index 100% rename from src/client/vsock_client.rs rename to pbs-client/src/vsock_client.rs diff --git a/pbs-tools/Cargo.toml b/pbs-tools/Cargo.toml index ab80666c..e149f048 100644 --- a/pbs-tools/Cargo.toml +++ b/pbs-tools/Cargo.toml @@ -9,6 +9,10 @@ description = "common tools used throughout pbs" [dependencies] anyhow = "1.0" base64 = "0.12" +bytes = "1.0" +crc32fast = "1" +endian_trait = { version = "0.6", features = ["arrays"] } +flate2 = "1.0" foreign-types = "0.3" futures = "0.3" lazy_static = "1.4" @@ -21,9 +25,10 @@ regex = "1.2" serde = "1.0" serde_json = "1.0" # rt-multi-thread is required for block_in_place -tokio = { version = "1.6", features = [ "rt", "rt-multi-thread", "sync" ] } +tokio = { version = "1.6", features = [ "fs", "io-util", "rt", "rt-multi-thread", "sync" ] } url = "2.1" +walkdir = "2" -proxmox = { version = "0.11.5", default-features = false, features = [] } +proxmox = { version = "0.11.5", default-features = false, features = [ "tokio" ] } pbs-buildcfg = { path = "../pbs-buildcfg" } diff --git a/src/tools/acl.rs b/pbs-tools/src/acl.rs similarity index 100% rename from src/tools/acl.rs rename to pbs-tools/src/acl.rs diff --git a/pbs-tools/src/compression.rs b/pbs-tools/src/compression.rs new file mode 100644 index 00000000..e2b6d795 --- /dev/null +++ b/pbs-tools/src/compression.rs @@ -0,0 +1,194 @@ +use std::io; +use std::pin::Pin; +use std::task::{Context, Poll}; + +use anyhow::Error; +use bytes::Bytes; +use flate2::{Compress, Compression, FlushCompress}; +use futures::ready; +use futures::stream::Stream; +use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; + +use proxmox::io_format_err; +use proxmox::tools::byte_buffer::ByteBuffer; + +const BUFFER_SIZE: usize = 8192; + +pub enum Level { + Fastest, + Best, + Default, + Precise(u32), +} + +#[derive(Eq, PartialEq)] +enum EncoderState { + Reading, + Writing, + Flushing, + Finished, +} + +pub struct DeflateEncoder { + inner: T, + compressor: Compress, + buffer: ByteBuffer, + input_buffer: Bytes, + state: EncoderState, +} + +impl DeflateEncoder { + pub fn new(inner: T) -> Self { + Self::with_quality(inner, Level::Default) + } + + pub fn with_quality(inner: T, level: Level) -> Self { + let level = match level { + Level::Fastest => Compression::fast(), + Level::Best => Compression::best(), + Level::Default => Compression::new(3), + Level::Precise(val) => Compression::new(val), + }; + + Self { + inner, + compressor: Compress::new(level, false), + buffer: ByteBuffer::with_capacity(BUFFER_SIZE), + input_buffer: Bytes::new(), + state: EncoderState::Reading, + } + } + + pub fn total_in(&self) -> u64 { + self.compressor.total_in() + } + + pub fn total_out(&self) -> u64 { + self.compressor.total_out() + } + + pub fn into_inner(self) -> T { + self.inner + } + + fn encode( + &mut self, + inbuf: &[u8], + flush: FlushCompress, + ) -> Result<(usize, flate2::Status), io::Error> { + let old_in = self.compressor.total_in(); + let old_out = self.compressor.total_out(); + let res = self + .compressor + .compress(&inbuf[..], self.buffer.get_free_mut_slice(), flush)?; + let new_in = (self.compressor.total_in() - old_in) as usize; + let new_out = (self.compressor.total_out() - old_out) as usize; + self.buffer.add_size(new_out); + + Ok((new_in, res)) + } +} + +impl DeflateEncoder> { + // assume small files + pub async fn compress_vec(&mut self, reader: &mut R, size_hint: usize) -> Result<(), Error> + where + R: AsyncRead + Unpin, + { + let mut buffer = Vec::with_capacity(size_hint); + reader.read_to_end(&mut buffer).await?; + self.inner.reserve(size_hint); // should be enough since we want smalller files + self.compressor.compress_vec(&buffer[..], &mut self.inner, FlushCompress::Finish)?; + Ok(()) + } +} + +impl DeflateEncoder { + pub async fn compress(&mut self, reader: &mut R) -> Result<(), Error> + where + R: AsyncRead + Unpin, + { + let mut buffer = ByteBuffer::with_capacity(BUFFER_SIZE); + let mut eof = false; + loop { + if !eof && !buffer.is_full() { + let read = buffer.read_from_async(reader).await?; + if read == 0 { + eof = true; + } + } + let (read, _res) = self.encode(&buffer[..], FlushCompress::None)?; + buffer.consume(read); + + self.inner.write_all(&self.buffer[..]).await?; + self.buffer.clear(); + + if buffer.is_empty() && eof { + break; + } + } + + loop { + let (_read, res) = self.encode(&[][..], FlushCompress::Finish)?; + self.inner.write_all(&self.buffer[..]).await?; + self.buffer.clear(); + if res == flate2::Status::StreamEnd { + break; + } + } + + Ok(()) + } +} + +impl Stream for DeflateEncoder +where + T: Stream> + Unpin, + O: Into +{ + type Item = Result; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = self.get_mut(); + + loop { + match this.state { + EncoderState::Reading => { + if let Some(res) = ready!(Pin::new(&mut this.inner).poll_next(cx)) { + let buf = res?; + this.input_buffer = buf.into(); + this.state = EncoderState::Writing; + } else { + this.state = EncoderState::Flushing; + } + } + EncoderState::Writing => { + if this.input_buffer.is_empty() { + return Poll::Ready(Some(Err(io_format_err!("empty input during write")))); + } + let mut buf = this.input_buffer.split_off(0); + let (read, res) = this.encode(&buf[..], FlushCompress::None)?; + this.input_buffer = buf.split_off(read); + if this.input_buffer.is_empty() { + this.state = EncoderState::Reading; + } + if this.buffer.is_full() || res == flate2::Status::BufError { + let bytes = this.buffer.remove_data(this.buffer.len()).to_vec(); + return Poll::Ready(Some(Ok(bytes.into()))); + } + } + EncoderState::Flushing => { + let (_read, res) = this.encode(&[][..], FlushCompress::Finish)?; + if !this.buffer.is_empty() { + let bytes = this.buffer.remove_data(this.buffer.len()).to_vec(); + return Poll::Ready(Some(Ok(bytes.into()))); + } + if res == flate2::Status::StreamEnd { + this.state = EncoderState::Finished; + } + } + EncoderState::Finished => return Poll::Ready(None), + } + } + } +} diff --git a/pbs-tools/src/fs.rs b/pbs-tools/src/fs.rs index 9f8325e1..1d2699ca 100644 --- a/pbs-tools/src/fs.rs +++ b/pbs-tools/src/fs.rs @@ -1,13 +1,15 @@ //! File system helper utilities. use std::borrow::{Borrow, BorrowMut}; +use std::collections::HashMap; +use std::hash::BuildHasher; use std::ops::{Deref, DerefMut}; use std::os::unix::io::{AsRawFd, RawFd}; use anyhow::{bail, format_err, Error}; use nix::dir; use nix::dir::Dir; -use nix::fcntl::OFlag; +use nix::fcntl::{AtFlags, OFlag}; use nix::sys::stat::Mode; use regex::Regex; @@ -344,3 +346,58 @@ fn do_lock_dir_noblock( Ok(handle) } + +pub fn complete_file_name(arg: &str, _param: &HashMap) -> Vec +where + S: BuildHasher, +{ + let mut result = vec![]; + + let mut dirname = std::path::PathBuf::from(if arg.is_empty() { "./" } else { arg }); + + let is_dir = match nix::sys::stat::fstatat(libc::AT_FDCWD, &dirname, AtFlags::empty()) { + Ok(stat) => (stat.st_mode & libc::S_IFMT) == libc::S_IFDIR, + Err(_) => false, + }; + + if !is_dir { + if let Some(parent) = dirname.parent() { + dirname = parent.to_owned(); + } + } + + let mut dir = + match nix::dir::Dir::openat(libc::AT_FDCWD, &dirname, OFlag::O_DIRECTORY, Mode::empty()) { + Ok(d) => d, + Err(_) => return result, + }; + + for item in dir.iter() { + if let Ok(entry) = item { + if let Ok(name) = entry.file_name().to_str() { + if name == "." || name == ".." { + continue; + } + let mut newpath = dirname.clone(); + newpath.push(name); + + if let Ok(stat) = + nix::sys::stat::fstatat(libc::AT_FDCWD, &newpath, AtFlags::empty()) + { + if (stat.st_mode & libc::S_IFMT) == libc::S_IFDIR { + newpath.push(""); + if let Some(newpath) = newpath.to_str() { + result.push(newpath.to_owned()); + } + continue; + } + } + if let Some(newpath) = newpath.to_str() { + result.push(newpath.to_owned()); + } + } + } + } + + result +} diff --git a/pbs-tools/src/lib.rs b/pbs-tools/src/lib.rs index c9d95dd9..075b066b 100644 --- a/pbs-tools/src/lib.rs +++ b/pbs-tools/src/lib.rs @@ -1,11 +1,14 @@ +pub mod acl; pub mod auth; pub mod borrow; pub mod broadcast_future; pub mod cert; +pub mod compression; pub mod format; pub mod fs; pub mod json; pub mod nom; +pub mod ops; pub mod percent_encoding; pub mod process_locker; pub mod sha; @@ -13,6 +16,8 @@ pub mod str; pub mod sync; pub mod ticket; pub mod tokio; +pub mod xattr; +pub mod zip; mod command; pub use command::{command_output, command_output_as_string, run_command}; diff --git a/pbs-tools/src/ops.rs b/pbs-tools/src/ops.rs new file mode 100644 index 00000000..49d0212e --- /dev/null +++ b/pbs-tools/src/ops.rs @@ -0,0 +1,12 @@ +//! std::ops extensions + +/// Modeled after the nightly `std::ops::ControlFlow`. +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum ControlFlow { + Continue(C), + Break(B), +} + +impl ControlFlow { + pub const CONTINUE: ControlFlow = ControlFlow::Continue(()); +} diff --git a/pbs-tools/src/str.rs b/pbs-tools/src/str.rs index 9b2d66ef..387fe5fe 100644 --- a/pbs-tools/src/str.rs +++ b/pbs-tools/src/str.rs @@ -15,3 +15,13 @@ pub fn join>(data: &[S], sep: char) -> String { list } +pub fn strip_ascii_whitespace(line: &[u8]) -> &[u8] { + let line = match line.iter().position(|&b| !b.is_ascii_whitespace()) { + Some(n) => &line[n..], + None => return &[], + }; + match line.iter().rev().position(|&b| !b.is_ascii_whitespace()) { + Some(n) => &line[..(line.len() - n)], + None => &[], + } +} diff --git a/src/tools/xattr.rs b/pbs-tools/src/xattr.rs similarity index 100% rename from src/tools/xattr.rs rename to pbs-tools/src/xattr.rs diff --git a/src/tools/zip.rs b/pbs-tools/src/zip.rs similarity index 99% rename from src/tools/zip.rs rename to pbs-tools/src/zip.rs index a9ef9b6d..c0bf76cd 100644 --- a/src/tools/zip.rs +++ b/pbs-tools/src/zip.rs @@ -22,7 +22,7 @@ use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt, ReadBuf}; use crc32fast::Hasher; use proxmox::tools::time::gmtime; -use crate::tools::compression::{DeflateEncoder, Level}; +use crate::compression::{DeflateEncoder, Level}; const LOCAL_FH_SIG: u32 = 0x04034B50; const LOCAL_FF_SIG: u32 = 0x08074B50; diff --git a/src/api2/access/user.rs b/src/api2/access/user.rs index e080d57a..70481ffb 100644 --- a/src/api2/access/user.rs +++ b/src/api2/access/user.rs @@ -10,7 +10,11 @@ use proxmox::api::router::SubdirMap; use proxmox::api::schema::{Schema, StringSchema}; use proxmox::tools::fs::open_file_locked; -use crate::api2::types::*; +use pbs_api_types::{ + PASSWORD_FORMAT, PROXMOX_CONFIG_DIGEST_SCHEMA, SINGLE_LINE_COMMENT_SCHEMA, Authid, + Tokenname, UserWithTokens, Userid, +}; + use crate::config::user; use crate::config::token_shadow; use crate::config::acl::{PRIV_SYS_AUDIT, PRIV_PERMISSIONS_MODIFY}; @@ -22,77 +26,16 @@ pub const PBS_PASSWORD_SCHEMA: Schema = StringSchema::new("User Password.") .max_length(64) .schema(); -#[api( - properties: { - userid: { - type: Userid, - }, - comment: { - optional: true, - schema: SINGLE_LINE_COMMENT_SCHEMA, - }, - enable: { - optional: true, - schema: user::ENABLE_USER_SCHEMA, - }, - expire: { - optional: true, - schema: user::EXPIRE_USER_SCHEMA, - }, - firstname: { - optional: true, - schema: user::FIRST_NAME_SCHEMA, - }, - lastname: { - schema: user::LAST_NAME_SCHEMA, - optional: true, - }, - email: { - schema: user::EMAIL_SCHEMA, - optional: true, - }, - tokens: { - type: Array, - optional: true, - description: "List of user's API tokens.", - items: { - type: user::ApiToken - }, - }, - } -)] -#[derive(Serialize,Deserialize)] -/// User properties with added list of ApiTokens -pub struct UserWithTokens { - pub userid: Userid, - #[serde(skip_serializing_if="Option::is_none")] - pub comment: Option, - #[serde(skip_serializing_if="Option::is_none")] - pub enable: Option, - #[serde(skip_serializing_if="Option::is_none")] - pub expire: Option, - #[serde(skip_serializing_if="Option::is_none")] - pub firstname: Option, - #[serde(skip_serializing_if="Option::is_none")] - pub lastname: Option, - #[serde(skip_serializing_if="Option::is_none")] - pub email: Option, - #[serde(skip_serializing_if="Vec::is_empty", default)] - pub tokens: Vec, -} - -impl UserWithTokens { - fn new(user: user::User) -> Self { - Self { - userid: user.userid, - comment: user.comment, - enable: user.enable, - expire: user.expire, - firstname: user.firstname, - lastname: user.lastname, - email: user.email, - tokens: Vec::new(), - } +fn new_user_with_tokens(user: user::User) -> UserWithTokens { + UserWithTokens { + userid: user.userid, + comment: user.comment, + enable: user.enable, + expire: user.expire, + firstname: user.firstname, + lastname: user.lastname, + email: user.email, + tokens: Vec::new(), } } @@ -165,13 +108,13 @@ pub fn list_users( }); iter .map(|user: user::User| { - let mut user = UserWithTokens::new(user); + let mut user = new_user_with_tokens(user); user.tokens = user_to_tokens.remove(&user.userid).unwrap_or_default(); user }) .collect() } else { - iter.map(UserWithTokens::new) + iter.map(new_user_with_tokens) .collect() }; diff --git a/src/api2/admin/datastore.rs b/src/api2/admin/datastore.rs index 60941501..470df9b6 100644 --- a/src/api2/admin/datastore.rs +++ b/src/api2/admin/datastore.rs @@ -26,13 +26,14 @@ use proxmox::{http_err, identity, list_subdirs_api_method, sortable}; use pxar::accessor::aio::Accessor; use pxar::EntryKind; +use pbs_client::pxar::create_zip; + use crate::api2::types::*; use crate::api2::node::rrd::create_value_from_rrd; use crate::api2::helpers; use crate::backup::*; use crate::config::datastore; use crate::config::cached_user_info::CachedUserInfo; -use crate::pxar::create_zip; use crate::server::{jobstate::Job, WorkerTask}; use crate::tools::{ diff --git a/src/api2/backup/mod.rs b/src/api2/backup/mod.rs index d302b2f7..eafff189 100644 --- a/src/api2/backup/mod.rs +++ b/src/api2/backup/mod.rs @@ -13,6 +13,7 @@ use proxmox::api::router::SubdirMap; use proxmox::api::schema::*; use pbs_tools::fs::lock_dir_noblock_shared; +use pbs_datastore::PROXMOX_BACKUP_PROTOCOL_ID_V1; use crate::tools; use crate::server::{WorkerTask, H2Service}; diff --git a/src/api2/config/remote.rs b/src/api2/config/remote.rs index 446a2604..8eacbc85 100644 --- a/src/api2/config/remote.rs +++ b/src/api2/config/remote.rs @@ -6,8 +6,9 @@ use proxmox::api::{api, ApiMethod, Router, RpcEnvironment, Permission}; use proxmox::http_err; use proxmox::tools::fs::open_file_locked; +use pbs_client::{HttpClient, HttpClientOptions}; + use crate::api2::types::*; -use crate::client::{HttpClient, HttpClientOptions}; use crate::config::cached_user_info::CachedUserInfo; use crate::config::remote; use crate::config::acl::{PRIV_REMOTE_AUDIT, PRIV_REMOTE_MODIFY}; diff --git a/src/api2/pull.rs b/src/api2/pull.rs index 0998e9b8..4893c9fb 100644 --- a/src/api2/pull.rs +++ b/src/api2/pull.rs @@ -7,9 +7,10 @@ use futures::{select, future::FutureExt}; use proxmox::api::api; use proxmox::api::{ApiMethod, Router, RpcEnvironment, Permission}; +use pbs_client::{HttpClient, BackupRepository}; + use crate::server::{WorkerTask, jobstate::Job, pull::pull_store}; use crate::backup::DataStore; -use crate::client::{HttpClient, BackupRepository}; use crate::api2::types::{ DATASTORE_SCHEMA, REMOTE_ID_SCHEMA, REMOVE_VANISHED_BACKUPS_SCHEMA, Authid, }; diff --git a/src/api2/reader/mod.rs b/src/api2/reader/mod.rs index 81d92bf1..533bc88c 100644 --- a/src/api2/reader/mod.rs +++ b/src/api2/reader/mod.rs @@ -28,6 +28,7 @@ use proxmox::{ }; use pbs_tools::fs::lock_dir_noblock_shared; +use pbs_datastore::PROXMOX_BACKUP_READER_PROTOCOL_ID_V1; use crate::{ api2::{ diff --git a/src/api2/types/mod.rs b/src/api2/types/mod.rs index b8b42a5e..530ce904 100644 --- a/src/api2/types/mod.rs +++ b/src/api2/types/mod.rs @@ -107,14 +107,6 @@ pub const TAPE_ENCRYPTION_KEY_FINGERPRINT_SCHEMA: Schema = StringSchema::new( .format(&FINGERPRINT_SHA256_FORMAT) .schema(); -pub const PROXMOX_CONFIG_DIGEST_SCHEMA: Schema = StringSchema::new( - "Prevent changes if current configuration file has different \ - SHA256 digest. This can be used to prevent concurrent \ - modifications." -) - .format(&PVE_CONFIG_DIGEST_FORMAT) .schema(); - - pub const CHUNK_DIGEST_SCHEMA: Schema = StringSchema::new("Chunk digest (SHA256).") .format(&CHUNK_DIGEST_FORMAT) .schema(); diff --git a/src/backup/mod.rs b/src/backup/mod.rs index 4161b402..2f2c8426 100644 --- a/src/backup/mod.rs +++ b/src/backup/mod.rs @@ -5,20 +5,6 @@ use anyhow::{bail, Error}; // Note: .pcat1 => Proxmox Catalog Format version 1 pub const CATALOG_NAME: &str = "catalog.pcat1.didx"; -#[macro_export] -macro_rules! PROXMOX_BACKUP_PROTOCOL_ID_V1 { - () => { - "proxmox-backup-protocol-v1" - }; -} - -#[macro_export] -macro_rules! PROXMOX_BACKUP_READER_PROTOCOL_ID_V1 { - () => { - "proxmox-backup-reader-protocol-v1" - }; -} - /// Unix system user used by proxmox-backup-proxy pub const BACKUP_USER_NAME: &str = "backup"; /// Unix system group used by proxmox-backup-proxy @@ -102,9 +88,5 @@ pub use datastore::*; mod verify; pub use verify::*; -// Move to client -mod catalog_shell; -pub use catalog_shell::*; - mod cached_chunk_reader; pub use cached_chunk_reader::*; diff --git a/src/bin/dump-catalog-shell-cli.rs b/src/bin/dump-catalog-shell-cli.rs index 7dcb9dc1..37ce36ea 100644 --- a/src/bin/dump-catalog-shell-cli.rs +++ b/src/bin/dump-catalog-shell-cli.rs @@ -3,7 +3,7 @@ use anyhow::{Error}; use proxmox::api::format::*; use proxmox::api::cli::*; -use proxmox_backup::backup::catalog_shell_cli; +use pbs_client::catalog_shell::catalog_shell_cli; fn main() -> Result<(), Error> { diff --git a/src/bin/proxmox-backup-client.rs b/src/bin/proxmox-backup-client.rs index 3a8a42a0..11ad5417 100644 --- a/src/bin/proxmox-backup-client.rs +++ b/src/bin/proxmox-backup-client.rs @@ -27,38 +27,59 @@ use proxmox::{ }; use pxar::accessor::{MaybeReady, ReadAt, ReadAtOperation}; +use pbs_api_types::CryptMode; +use pbs_client::{ + BACKUP_SOURCE_SCHEMA, + BackupReader, + BackupRepository, + BackupSpecificationType, + BackupStats, + BackupWriter, + HttpClient, + PxarBackupStream, + RemoteChunkReader, + UploadOptions, + delete_ticket_info, + parse_backup_specification, + view_task_result, +}; +use pbs_client::catalog_shell::Shell; +use pbs_client::tools::{ + complete_archive_name, complete_auth_id, complete_backup_group, complete_backup_snapshot, + complete_backup_source, complete_chunk_size, complete_group_or_snapshot, + complete_img_archive_name, complete_pxar_archive_name, complete_repository, connect, + extract_repository_from_value, + key_source::{ + crypto_parameters, format_key_source, get_encryption_key_password, KEYFD_SCHEMA, + KEYFILE_SCHEMA, MASTER_PUBKEY_FD_SCHEMA, MASTER_PUBKEY_FILE_SCHEMA, + }, + CHUNK_SIZE_SCHEMA, REPO_URL_SCHEMA, +}; +use pbs_datastore::CryptConfig; +use pbs_datastore::backup_info::{BackupDir, BackupGroup}; use pbs_datastore::catalog::BackupCatalogWriter; +use pbs_datastore::dynamic_index::DynamicIndexReader; +use pbs_datastore::fixed_index::FixedIndexReader; +use pbs_datastore::index::IndexFile; +use pbs_datastore::manifest::{MANIFEST_BLOB_NAME, ArchiveType, BackupManifest, archive_type}; +use pbs_datastore::read_chunk::AsyncReadChunk; use pbs_tools::sync::StdChannelWriter; use pbs_tools::tokio::TokioWriterAdapter; use proxmox_backup::api2::types::*; use proxmox_backup::api2::version; -use proxmox_backup::client::*; use proxmox_backup::backup::{ - archive_type, decrypt_key, rsa_encrypt_key_config, verify_chunk_size, - ArchiveType, - AsyncReadChunk, - BackupDir, - BackupGroup, - BackupManifest, BufferedDynamicReader, CATALOG_NAME, CatalogReader, CatalogWriter, ChunkStream, - CryptConfig, - CryptMode, - DynamicIndexReader, ENCRYPTED_KEY_BLOB_NAME, FixedChunkStream, - FixedIndexReader, KeyConfig, - IndexFile, - MANIFEST_BLOB_NAME, - Shell, PruneOptions, }; use proxmox_backup::tools; @@ -66,19 +87,6 @@ use proxmox_backup::tools; mod proxmox_backup_client; use proxmox_backup_client::*; -pub mod proxmox_client_tools; -use proxmox_client_tools::{ - complete_archive_name, complete_auth_id, complete_backup_group, complete_backup_snapshot, - complete_backup_source, complete_chunk_size, complete_group_or_snapshot, - complete_img_archive_name, complete_pxar_archive_name, complete_repository, connect, - extract_repository_from_value, - key_source::{ - crypto_parameters, format_key_source, get_encryption_key_password, KEYFD_SCHEMA, - KEYFILE_SCHEMA, MASTER_PUBKEY_FD_SCHEMA, MASTER_PUBKEY_FILE_SCHEMA, - }, - CHUNK_SIZE_SCHEMA, REPO_URL_SCHEMA, -}; - fn record_repository(repo: &BackupRepository) { let base = match BaseDirectories::with_prefix("proxmox-backup") { @@ -172,7 +180,7 @@ async fn backup_directory>( archive_name: &str, chunk_size: Option, catalog: Arc>>>, - pxar_create_options: proxmox_backup::pxar::PxarCreateOptions, + pxar_create_options: pbs_client::pxar::PxarCreateOptions, upload_options: UploadOptions, ) -> Result { @@ -589,7 +597,7 @@ fn spawn_catalog_upload( type: Integer, description: "Max number of entries to hold in memory.", optional: true, - default: proxmox_backup::pxar::ENCODER_MAX_ENTRIES as isize, + default: pbs_client::pxar::ENCODER_MAX_ENTRIES as isize, }, "verbose": { type: Boolean, @@ -633,7 +641,7 @@ async fn create_backup( let include_dev = param["include-dev"].as_array(); let entries_max = param["entries-max"].as_u64() - .unwrap_or(proxmox_backup::pxar::ENCODER_MAX_ENTRIES as u64); + .unwrap_or(pbs_client::pxar::ENCODER_MAX_ENTRIES as u64); let empty = Vec::new(); let exclude_args = param["exclude"].as_array().unwrap_or(&empty); @@ -856,7 +864,7 @@ async fn create_backup( println!("Upload directory '{}' to '{}' as {}", filename, repo, target); catalog.lock().unwrap().start_directory(std::ffi::CString::new(target.as_str())?.as_c_str())?; - let pxar_options = proxmox_backup::pxar::PxarCreateOptions { + let pxar_options = pbs_client::pxar::PxarCreateOptions { device_set: devices.clone(), patterns: pattern_list.clone(), entries_max: entries_max as usize, @@ -1168,7 +1176,7 @@ async fn restore(param: Value) -> Result { let mut reader = BufferedDynamicReader::new(index, chunk_reader); - let options = proxmox_backup::pxar::PxarExtractOptions { + let options = pbs_client::pxar::PxarExtractOptions { match_list: &[], extract_match_default: true, allow_existing_dirs, @@ -1176,10 +1184,10 @@ async fn restore(param: Value) -> Result { }; if let Some(target) = target { - proxmox_backup::pxar::extract_archive( + pbs_client::pxar::extract_archive( pxar::decoder::Decoder::from_std(reader)?, Path::new(target), - proxmox_backup::pxar::Flags::DEFAULT, + pbs_client::pxar::Flags::DEFAULT, |path| { if verbose { println!("{:?}", path); @@ -1377,7 +1385,6 @@ async fn status(param: Value) -> Result { Ok(Value::Null) } -use proxmox_backup::client::RemoteChunkReader; /// This is a workaround until we have cleaned up the chunk/reader/... infrastructure for better /// async use! /// @@ -1424,13 +1431,13 @@ fn main() { .arg_param(&["backupspec"]) .completion_cb("repository", complete_repository) .completion_cb("backupspec", complete_backup_source) - .completion_cb("keyfile", tools::complete_file_name) - .completion_cb("master-pubkey-file", tools::complete_file_name) + .completion_cb("keyfile", pbs_tools::fs::complete_file_name) + .completion_cb("master-pubkey-file", pbs_tools::fs::complete_file_name) .completion_cb("chunk-size", complete_chunk_size); let benchmark_cmd_def = CliCommand::new(&API_METHOD_BENCHMARK) .completion_cb("repository", complete_repository) - .completion_cb("keyfile", tools::complete_file_name); + .completion_cb("keyfile", pbs_tools::fs::complete_file_name); let list_cmd_def = CliCommand::new(&API_METHOD_LIST_BACKUP_GROUPS) .completion_cb("repository", complete_repository); @@ -1443,7 +1450,7 @@ fn main() { .completion_cb("repository", complete_repository) .completion_cb("snapshot", complete_group_or_snapshot) .completion_cb("archive-name", complete_archive_name) - .completion_cb("target", tools::complete_file_name); + .completion_cb("target", pbs_tools::fs::complete_file_name); let prune_cmd_def = CliCommand::new(&API_METHOD_PRUNE) .arg_param(&["group"]) diff --git a/src/bin/proxmox-backup-manager.rs b/src/bin/proxmox-backup-manager.rs index 5c70d8c7..ca3ac1ef 100644 --- a/src/bin/proxmox-backup-manager.rs +++ b/src/bin/proxmox-backup-manager.rs @@ -6,12 +6,12 @@ use serde_json::{json, Value}; use proxmox::api::{api, cli::*, RpcEnvironment}; +use pbs_client::{connect_to_localhost, display_task_log, view_task_result}; use pbs_tools::percent_encoding::percent_encode_component; use proxmox_backup::tools; use proxmox_backup::config; use proxmox_backup::api2::{self, types::* }; -use proxmox_backup::client::*; use proxmox_backup::server::wait_for_local_worker; mod proxmox_backup_manager; diff --git a/src/bin/proxmox-file-restore.rs b/src/bin/proxmox-file-restore.rs index 17033cf5..17c1506a 100644 --- a/src/bin/proxmox-file-restore.rs +++ b/src/bin/proxmox-file-restore.rs @@ -16,18 +16,10 @@ use proxmox::api::{ use pxar::accessor::aio::Accessor; use pxar::decoder::aio::Decoder; -use proxmox_backup::api2::{helpers, types::ArchiveEntry}; -use proxmox_backup::backup::{ - decrypt_key, BackupDir, BufferedDynamicReader, CatalogReader, CryptConfig, CryptMode, - DirEntryAttribute, IndexFile, LocalDynamicReadAt, CATALOG_NAME, -}; -use proxmox_backup::client::{BackupReader, RemoteChunkReader}; -use proxmox_backup::pxar::{create_zip, extract_sub_dir, extract_sub_dir_seq}; -use proxmox_backup::tools; - -// use "pub" so rust doesn't complain about "unused" functions in the module -pub mod proxmox_client_tools; -use proxmox_client_tools::{ +use pbs_datastore::index::IndexFile; +use pbs_client::{BackupReader, RemoteChunkReader}; +use pbs_client::pxar::{create_zip, extract_sub_dir, extract_sub_dir_seq}; +use pbs_client::tools::{ complete_group_or_snapshot, complete_repository, connect, extract_repository_from_value, key_source::{ crypto_parameters_keep_fd, format_key_source, get_encryption_key_password, KEYFD_SCHEMA, @@ -36,6 +28,13 @@ use proxmox_client_tools::{ REPO_URL_SCHEMA, }; +use proxmox_backup::api2::{helpers, types::ArchiveEntry}; +use proxmox_backup::backup::{ + decrypt_key, BackupDir, BufferedDynamicReader, CatalogReader, CryptConfig, CryptMode, + DirEntryAttribute, LocalDynamicReadAt, CATALOG_NAME, +}; +use proxmox_backup::tools; + mod proxmox_file_restore; use proxmox_file_restore::*; @@ -456,7 +455,7 @@ fn main() { .arg_param(&["snapshot", "path", "target"]) .completion_cb("repository", complete_repository) .completion_cb("snapshot", complete_group_or_snapshot) - .completion_cb("target", tools::complete_file_name); + .completion_cb("target", pbs_tools::fs::complete_file_name); let status_cmd_def = CliCommand::new(&API_METHOD_STATUS); let stop_cmd_def = CliCommand::new(&API_METHOD_STOP) @@ -476,3 +475,15 @@ fn main() { Some(|future| pbs_runtime::main(future)), ); } + +/// Returns a runtime dir owned by the current user. +/// Note that XDG_RUNTIME_DIR is not always available, especially for non-login users like +/// "www-data", so we use a custom one in /run/proxmox-backup/ instead. +pub fn get_user_run_dir() -> Result { + let uid = nix::unistd::Uid::current(); + let mut path: std::path::PathBuf = pbs_buildcfg::PROXMOX_BACKUP_RUN_DIR.into(); + path.push(uid.to_string()); + tools::create_run_dir()?; + std::fs::create_dir_all(&path)?; + Ok(path) +} diff --git a/src/bin/proxmox-restore-daemon.rs b/src/bin/proxmox-restore-daemon.rs index 3e59ccfa..ebe8cef5 100644 --- a/src/bin/proxmox-restore-daemon.rs +++ b/src/bin/proxmox-restore-daemon.rs @@ -13,8 +13,9 @@ use std::sync::{Arc, Mutex}; use tokio::sync::mpsc; use tokio_stream::wrappers::ReceiverStream; +use pbs_client::DEFAULT_VSOCK_PORT; + use proxmox::api::RpcEnvironmentType; -use proxmox_backup::client::DEFAULT_VSOCK_PORT; use proxmox_backup::server::{rest::*, ApiConfig}; mod proxmox_restore_daemon; diff --git a/src/bin/proxmox-tape.rs b/src/bin/proxmox-tape.rs index e2e88781..ae3fb4f6 100644 --- a/src/bin/proxmox-tape.rs +++ b/src/bin/proxmox-tape.rs @@ -14,6 +14,7 @@ use proxmox::{ }, }; +use pbs_client::{connect_to_localhost, view_task_result}; use pbs_tools::format::{ HumanByte, render_epoch, @@ -21,10 +22,6 @@ use pbs_tools::format::{ }; use proxmox_backup::{ - client::{ - connect_to_localhost, - view_task_result, - }, api2::{ self, types::{ diff --git a/src/bin/proxmox_backup_client/benchmark.rs b/src/bin/proxmox_backup_client/benchmark.rs index c1673701..228464d2 100644 --- a/src/bin/proxmox_backup_client/benchmark.rs +++ b/src/bin/proxmox_backup_client/benchmark.rs @@ -18,6 +18,9 @@ use proxmox::api::{ router::ReturnType, }; +use pbs_client::tools::key_source::get_encryption_key_password; +use pbs_client::{BackupRepository, BackupWriter}; + use proxmox_backup::backup::{ load_and_decrypt_key, CryptConfig, @@ -25,8 +28,6 @@ use proxmox_backup::backup::{ DataChunkBuilder, }; -use proxmox_backup::client::*; - use crate::{ KEYFILE_SCHEMA, REPO_URL_SCHEMA, extract_repository_from_value, @@ -34,8 +35,6 @@ use crate::{ connect, }; -use crate::proxmox_client_tools::key_source::get_encryption_key_password; - #[api()] #[derive(Copy, Clone, Serialize)] /// Speed test result diff --git a/src/bin/proxmox_backup_client/catalog.rs b/src/bin/proxmox_backup_client/catalog.rs index f4b0a1d5..ce0fe809 100644 --- a/src/bin/proxmox_backup_client/catalog.rs +++ b/src/bin/proxmox_backup_client/catalog.rs @@ -7,9 +7,10 @@ use serde_json::Value; use proxmox::api::{api, cli::*}; -use proxmox_backup::tools; +use pbs_client::tools::key_source::get_encryption_key_password; +use pbs_client::{BackupReader, RemoteChunkReader}; -use proxmox_backup::client::*; +use proxmox_backup::tools; use crate::{ REPO_URL_SCHEMA, @@ -37,8 +38,6 @@ use crate::{ Shell, }; -use crate::proxmox_client_tools::key_source::get_encryption_key_password; - #[api( input: { properties: { @@ -219,9 +218,9 @@ async fn catalog_shell(param: Value) -> Result<(), Error> { let chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config.clone(), file_info.chunk_crypt_mode(), most_used); let reader = BufferedDynamicReader::new(index, chunk_reader); let archive_size = reader.archive_size(); - let reader: proxmox_backup::pxar::fuse::Reader = + let reader: pbs_client::pxar::fuse::Reader = Arc::new(BufferedDynamicReadAt::new(reader)); - let decoder = proxmox_backup::pxar::fuse::Accessor::new(reader, archive_size).await?; + let decoder = pbs_client::pxar::fuse::Accessor::new(reader, archive_size).await?; client.download(CATALOG_NAME, &mut tmpfile).await?; let index = DynamicIndexReader::new(tmpfile) diff --git a/src/bin/proxmox_backup_client/key.rs b/src/bin/proxmox_backup_client/key.rs index 49afcccb..7ca028bc 100644 --- a/src/bin/proxmox_backup_client/key.rs +++ b/src/bin/proxmox_backup_client/key.rs @@ -14,19 +14,18 @@ use proxmox::sys::linux::tty; use proxmox::tools::fs::{file_get_contents, replace_file, CreateOptions}; use pbs_datastore::{KeyInfo, Kdf}; +use pbs_client::tools::key_source::{ + find_default_encryption_key, find_default_master_pubkey, get_encryption_key_password, + place_default_encryption_key, place_default_master_pubkey, +}; + use proxmox_backup::{ api2::types::{RsaPubKeyInfo, PASSWORD_HINT_SCHEMA}, backup::{rsa_decrypt_key_config, KeyConfig}, - tools, tools::paperkey::{generate_paper_key, PaperkeyFormat}, }; -use crate::proxmox_client_tools::key_source::{ - find_default_encryption_key, find_default_master_pubkey, get_encryption_key_password, - place_default_encryption_key, place_default_master_pubkey, -}; - #[api( input: { properties: { @@ -458,35 +457,35 @@ fn paper_key( pub fn cli() -> CliCommandMap { let key_create_cmd_def = CliCommand::new(&API_METHOD_CREATE) .arg_param(&["path"]) - .completion_cb("path", tools::complete_file_name); + .completion_cb("path", pbs_tools::fs::complete_file_name); let key_import_with_master_key_cmd_def = CliCommand::new(&API_METHOD_IMPORT_WITH_MASTER_KEY) .arg_param(&["master-keyfile"]) - .completion_cb("master-keyfile", tools::complete_file_name) + .completion_cb("master-keyfile", pbs_tools::fs::complete_file_name) .arg_param(&["encrypted-keyfile"]) - .completion_cb("encrypted-keyfile", tools::complete_file_name) + .completion_cb("encrypted-keyfile", pbs_tools::fs::complete_file_name) .arg_param(&["path"]) - .completion_cb("path", tools::complete_file_name); + .completion_cb("path", pbs_tools::fs::complete_file_name); let key_change_passphrase_cmd_def = CliCommand::new(&API_METHOD_CHANGE_PASSPHRASE) .arg_param(&["path"]) - .completion_cb("path", tools::complete_file_name); + .completion_cb("path", pbs_tools::fs::complete_file_name); let key_create_master_key_cmd_def = CliCommand::new(&API_METHOD_CREATE_MASTER_KEY); let key_import_master_pubkey_cmd_def = CliCommand::new(&API_METHOD_IMPORT_MASTER_PUBKEY) .arg_param(&["path"]) - .completion_cb("path", tools::complete_file_name); + .completion_cb("path", pbs_tools::fs::complete_file_name); let key_show_master_pubkey_cmd_def = CliCommand::new(&API_METHOD_SHOW_MASTER_PUBKEY) .arg_param(&["path"]) - .completion_cb("path", tools::complete_file_name); + .completion_cb("path", pbs_tools::fs::complete_file_name); let key_show_cmd_def = CliCommand::new(&API_METHOD_SHOW_KEY) .arg_param(&["path"]) - .completion_cb("path", tools::complete_file_name); + .completion_cb("path", pbs_tools::fs::complete_file_name); let paper_key_cmd_def = CliCommand::new(&API_METHOD_PAPER_KEY) .arg_param(&["path"]) - .completion_cb("path", tools::complete_file_name); + .completion_cb("path", pbs_tools::fs::complete_file_name); CliCommandMap::new() .insert("create", key_create_cmd_def) diff --git a/src/bin/proxmox_backup_client/mount.rs b/src/bin/proxmox_backup_client/mount.rs index 21f78c32..44ba09a8 100644 --- a/src/bin/proxmox_backup_client/mount.rs +++ b/src/bin/proxmox_backup_client/mount.rs @@ -17,6 +17,9 @@ use proxmox::{sortable, identity}; use proxmox::api::{ApiHandler, ApiMethod, RpcEnvironment, schema::*, cli::*}; use proxmox::tools::fd::Fd; +use pbs_client::tools::key_source::get_encryption_key_password; +use pbs_client::{BackupReader, RemoteChunkReader}; + use proxmox_backup::tools; use proxmox_backup::backup::{ load_and_decrypt_key, @@ -28,8 +31,6 @@ use proxmox_backup::backup::{ CachedChunkReader, }; -use proxmox_backup::client::*; - use crate::{ REPO_URL_SCHEMA, extract_repository_from_value, @@ -43,8 +44,6 @@ use crate::{ BufferedDynamicReadAt, }; -use crate::proxmox_client_tools::key_source::get_encryption_key_password; - #[sortable] const API_METHOD_MOUNT: ApiMethod = ApiMethod::new( &ApiHandler::Sync(&mount), @@ -98,7 +97,7 @@ pub fn mount_cmd_def() -> CliCommand { .completion_cb("repository", complete_repository) .completion_cb("snapshot", complete_group_or_snapshot) .completion_cb("archive-name", complete_pxar_archive_name) - .completion_cb("target", tools::complete_file_name) + .completion_cb("target", pbs_tools::fs::complete_file_name) } pub fn map_cmd_def() -> CliCommand { @@ -257,11 +256,11 @@ async fn mount_do(param: Value, pipe: Option) -> Result { let chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config, file_info.chunk_crypt_mode(), most_used); let reader = BufferedDynamicReader::new(index, chunk_reader); let archive_size = reader.archive_size(); - let reader: proxmox_backup::pxar::fuse::Reader = + let reader: pbs_client::pxar::fuse::Reader = Arc::new(BufferedDynamicReadAt::new(reader)); - let decoder = proxmox_backup::pxar::fuse::Accessor::new(reader, archive_size).await?; + let decoder = pbs_client::pxar::fuse::Accessor::new(reader, archive_size).await?; - let session = proxmox_backup::pxar::fuse::Session::mount( + let session = pbs_client::pxar::fuse::Session::mount( decoder, &options, false, diff --git a/src/bin/proxmox_backup_client/snapshot.rs b/src/bin/proxmox_backup_client/snapshot.rs index 7deb6647..ae92e688 100644 --- a/src/bin/proxmox_backup_client/snapshot.rs +++ b/src/bin/proxmox_backup_client/snapshot.rs @@ -8,6 +8,8 @@ use proxmox::{ tools::fs::file_get_contents, }; +use pbs_client::tools::key_source::get_encryption_key_password; + use proxmox_backup::{ tools, api2::types::*, @@ -35,8 +37,6 @@ use crate::{ record_repository, }; -use crate::proxmox_client_tools::key_source::get_encryption_key_password; - #[api( input: { properties: { @@ -412,8 +412,8 @@ pub fn snapshot_mgtm_cli() -> CliCommandMap { CliCommand::new(&API_METHOD_UPLOAD_LOG) .arg_param(&["snapshot", "logfile"]) .completion_cb("snapshot", complete_backup_snapshot) - .completion_cb("logfile", tools::complete_file_name) - .completion_cb("keyfile", tools::complete_file_name) + .completion_cb("logfile", pbs_tools::fs::complete_file_name) + .completion_cb("keyfile", pbs_tools::fs::complete_file_name) .completion_cb("repository", complete_repository) ) } diff --git a/src/bin/proxmox_backup_client/task.rs b/src/bin/proxmox_backup_client/task.rs index e6fcc74e..a65d5a3b 100644 --- a/src/bin/proxmox_backup_client/task.rs +++ b/src/bin/proxmox_backup_client/task.rs @@ -4,10 +4,10 @@ use serde_json::{json, Value}; use proxmox::api::{api, cli::*}; use pbs_tools::percent_encoding::percent_encode_component; +use pbs_client::display_task_log; use proxmox_backup::tools; -use proxmox_backup::client::*; use proxmox_backup::api2::types::UPID_SCHEMA; use crate::{ diff --git a/src/bin/proxmox_backup_manager/datastore.rs b/src/bin/proxmox_backup_manager/datastore.rs index 7cbd8805..a2436786 100644 --- a/src/bin/proxmox_backup_manager/datastore.rs +++ b/src/bin/proxmox_backup_manager/datastore.rs @@ -3,12 +3,10 @@ use serde_json::Value; use proxmox::api::{api, cli::*, RpcEnvironment, ApiHandler}; +use pbs_client::{connect_to_localhost, view_task_result}; + use proxmox_backup::config; use proxmox_backup::api2::{self, types::* }; -use proxmox_backup::client::{ - connect_to_localhost, - view_task_result, -}; use proxmox_backup::config::datastore::DIR_NAME_SCHEMA; #[api( diff --git a/src/bin/proxmox_file_restore/block_driver.rs b/src/bin/proxmox_file_restore/block_driver.rs index ba9794e3..700c681e 100644 --- a/src/bin/proxmox_file_restore/block_driver.rs +++ b/src/bin/proxmox_file_restore/block_driver.rs @@ -1,19 +1,20 @@ //! Abstraction layer over different methods of accessing a block backup -use anyhow::{bail, Error}; -use serde::{Deserialize, Serialize}; -use serde_json::{json, Value}; - use std::collections::HashMap; use std::future::Future; use std::hash::BuildHasher; use std::pin::Pin; -use proxmox_backup::backup::{BackupDir, BackupManifest}; -use proxmox_backup::api2::types::ArchiveEntry; -use proxmox_backup::client::BackupRepository; +use anyhow::{bail, Error}; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; use proxmox::api::{api, cli::*}; +use pbs_client::BackupRepository; + +use proxmox_backup::backup::{BackupDir, BackupManifest}; +use proxmox_backup::api2::types::ArchiveEntry; + use super::block_driver_qemu::QemuBlockDriver; /// Contains details about a snapshot that is to be accessed by block file restore diff --git a/src/bin/proxmox_file_restore/block_driver_qemu.rs b/src/bin/proxmox_file_restore/block_driver_qemu.rs index 46d91198..be1476bd 100644 --- a/src/bin/proxmox_file_restore/block_driver_qemu.rs +++ b/src/bin/proxmox_file_restore/block_driver_qemu.rs @@ -1,21 +1,23 @@ //! Block file access via a small QEMU restore VM using the PBS block driver in QEMU +use std::collections::HashMap; +use std::fs::{File, OpenOptions}; +use std::io::{prelude::*, SeekFrom}; + use anyhow::{bail, Error}; use futures::FutureExt; use serde::{Deserialize, Serialize}; use serde_json::json; -use std::collections::HashMap; -use std::fs::{File, OpenOptions}; -use std::io::{prelude::*, SeekFrom}; - use proxmox::tools::fs::lock_file; + +use pbs_client::{DEFAULT_VSOCK_PORT, BackupRepository, VsockClient}; + use proxmox_backup::api2::types::ArchiveEntry; use proxmox_backup::backup::BackupDir; -use proxmox_backup::client::*; use proxmox_backup::tools; use super::block_driver::*; -use crate::proxmox_client_tools::get_user_run_dir; +use crate::get_user_run_dir; const RESTORE_VM_MAP: &str = "restore-vm-map.json"; diff --git a/src/bin/proxmox_file_restore/qemu_helper.rs b/src/bin/proxmox_file_restore/qemu_helper.rs index 83e772cb..80e89102 100644 --- a/src/bin/proxmox_file_restore/qemu_helper.rs +++ b/src/bin/proxmox_file_restore/qemu_helper.rs @@ -13,8 +13,9 @@ use nix::unistd::Pid; use proxmox::tools::fs::{create_path, file_read_string, make_tmp_file, CreateOptions}; +use pbs_client::{VsockClient, DEFAULT_VSOCK_PORT}; + use proxmox_backup::backup::backup_user; -use proxmox_backup::client::{VsockClient, DEFAULT_VSOCK_PORT}; use proxmox_backup::tools; use super::SnapRestoreDetails; diff --git a/src/bin/proxmox_restore_daemon/api.rs b/src/bin/proxmox_restore_daemon/api.rs index b3721160..fbbda13c 100644 --- a/src/bin/proxmox_restore_daemon/api.rs +++ b/src/bin/proxmox_restore_daemon/api.rs @@ -19,12 +19,13 @@ use proxmox::api::{ }; use proxmox::{identity, list_subdirs_api_method, sortable}; +use pbs_client::pxar::{create_archive, Flags, PxarCreateOptions, ENCODER_MAX_ENTRIES}; use pbs_tools::fs::read_subdir; +use pbs_tools::zip::zip_directory; use proxmox_backup::api2::types::*; use proxmox_backup::backup::DirEntryAttribute; -use proxmox_backup::pxar::{create_archive, Flags, PxarCreateOptions, ENCODER_MAX_ENTRIES}; -use proxmox_backup::tools::{self, zip::zip_directory}; +use proxmox_backup::tools; use pxar::encoder::aio::TokioWriter; diff --git a/src/bin/proxmox_tape/backup_job.rs b/src/bin/proxmox_tape/backup_job.rs index 6046c3d2..d9b25f20 100644 --- a/src/bin/proxmox_tape/backup_job.rs +++ b/src/bin/proxmox_tape/backup_job.rs @@ -3,12 +3,10 @@ use serde_json::Value; use proxmox::api::{api, cli::*, RpcEnvironment, ApiHandler}; +use pbs_client::{connect_to_localhost, view_task_result}; + use proxmox_backup::{ config, - client::{ - connect_to_localhost, - view_task_result, - }, api2::{ self, types::*, diff --git a/src/bin/pxar.rs b/src/bin/pxar.rs index 34cae30c..e4eac0d7 100644 --- a/src/bin/pxar.rs +++ b/src/bin/pxar.rs @@ -12,13 +12,11 @@ use futures::select; use tokio::signal::unix::{signal, SignalKind}; use pathpatterns::{MatchEntry, MatchType, PatternFlag}; +use pbs_client::pxar::{fuse, format_single_line_entry, ENCODER_MAX_ENTRIES, Flags, PxarExtractOptions}; use proxmox::api::cli::*; use proxmox::api::api; -use proxmox_backup::tools; -use proxmox_backup::pxar::{fuse, format_single_line_entry, ENCODER_MAX_ENTRIES, Flags, PxarExtractOptions}; - fn extract_archive_from_reader( reader: &mut R, target: &str, @@ -26,8 +24,7 @@ fn extract_archive_from_reader( verbose: bool, options: PxarExtractOptions, ) -> Result<(), Error> { - - proxmox_backup::pxar::extract_archive( + pbs_client::pxar::extract_archive( pxar::decoder::Decoder::from_std(reader)?, Path::new(target), feature_flags, @@ -327,7 +324,7 @@ async fn create_archive( Some(HashSet::new()) }; - let options = proxmox_backup::pxar::PxarCreateOptions { + let options = pbs_client::pxar::PxarCreateOptions { entries_max: entries_max as usize, device_set, patterns, @@ -372,7 +369,7 @@ async fn create_archive( } let writer = pxar::encoder::sync::StandardWriter::new(writer); - proxmox_backup::pxar::create_archive( + pbs_client::pxar::create_archive( dir, writer, feature_flags, @@ -464,29 +461,29 @@ fn main() { "create", CliCommand::new(&API_METHOD_CREATE_ARCHIVE) .arg_param(&["archive", "source"]) - .completion_cb("archive", tools::complete_file_name) - .completion_cb("source", tools::complete_file_name), + .completion_cb("archive", pbs_tools::fs::complete_file_name) + .completion_cb("source", pbs_tools::fs::complete_file_name), ) .insert( "extract", CliCommand::new(&API_METHOD_EXTRACT_ARCHIVE) .arg_param(&["archive", "target"]) - .completion_cb("archive", tools::complete_file_name) - .completion_cb("target", tools::complete_file_name) - .completion_cb("files-from", tools::complete_file_name), + .completion_cb("archive", pbs_tools::fs::complete_file_name) + .completion_cb("target", pbs_tools::fs::complete_file_name) + .completion_cb("files-from", pbs_tools::fs::complete_file_name), ) .insert( "mount", CliCommand::new(&API_METHOD_MOUNT_ARCHIVE) .arg_param(&["archive", "mountpoint"]) - .completion_cb("archive", tools::complete_file_name) - .completion_cb("mountpoint", tools::complete_file_name), + .completion_cb("archive", pbs_tools::fs::complete_file_name) + .completion_cb("mountpoint", pbs_tools::fs::complete_file_name), ) .insert( "list", CliCommand::new(&API_METHOD_DUMP_ARCHIVE) .arg_param(&["archive"]) - .completion_cb("archive", tools::complete_file_name), + .completion_cb("archive", pbs_tools::fs::complete_file_name), ); let rpcenv = CliEnvironment::new(); diff --git a/src/config/user.rs b/src/config/user.rs index bdec5fc1..1406e386 100644 --- a/src/config/user.rs +++ b/src/config/user.rs @@ -3,10 +3,8 @@ use std::sync::{Arc, RwLock}; use anyhow::{bail, Error}; use lazy_static::lazy_static; -use serde::{Serialize, Deserialize}; use proxmox::api::{ - api, schema::*, section_config::{ SectionConfig, @@ -17,154 +15,18 @@ use proxmox::api::{ use proxmox::tools::{fs::replace_file, fs::CreateOptions}; -use crate::api2::types::*; +use pbs_api_types::{Authid, Userid}; +pub use pbs_api_types::{ApiToken, User}; +pub use pbs_api_types::{ + EMAIL_SCHEMA, ENABLE_USER_SCHEMA, EXPIRE_USER_SCHEMA, FIRST_NAME_SCHEMA, LAST_NAME_SCHEMA, +}; + use crate::tools::Memcom; lazy_static! { pub static ref CONFIG: SectionConfig = init(); } -pub const ENABLE_USER_SCHEMA: Schema = BooleanSchema::new( - "Enable the account (default). You can set this to '0' to disable the account.") - .default(true) - .schema(); - -pub const EXPIRE_USER_SCHEMA: Schema = IntegerSchema::new( - "Account expiration date (seconds since epoch). '0' means no expiration date.") - .default(0) - .minimum(0) - .schema(); - -pub const FIRST_NAME_SCHEMA: Schema = StringSchema::new("First name.") - .format(&SINGLE_LINE_COMMENT_FORMAT) - .min_length(2) - .max_length(64) - .schema(); - -pub const LAST_NAME_SCHEMA: Schema = StringSchema::new("Last name.") - .format(&SINGLE_LINE_COMMENT_FORMAT) - .min_length(2) - .max_length(64) - .schema(); - -pub const EMAIL_SCHEMA: Schema = StringSchema::new("E-Mail Address.") - .format(&SINGLE_LINE_COMMENT_FORMAT) - .min_length(2) - .max_length(64) - .schema(); - -#[api( - properties: { - tokenid: { - schema: PROXMOX_TOKEN_ID_SCHEMA, - }, - comment: { - optional: true, - schema: SINGLE_LINE_COMMENT_SCHEMA, - }, - enable: { - optional: true, - schema: ENABLE_USER_SCHEMA, - }, - expire: { - optional: true, - schema: EXPIRE_USER_SCHEMA, - }, - } -)] -#[derive(Serialize,Deserialize)] -/// ApiToken properties. -pub struct ApiToken { - pub tokenid: Authid, - #[serde(skip_serializing_if="Option::is_none")] - pub comment: Option, - #[serde(skip_serializing_if="Option::is_none")] - pub enable: Option, - #[serde(skip_serializing_if="Option::is_none")] - pub expire: Option, -} - -impl ApiToken { - - pub fn is_active(&self) -> bool { - if !self.enable.unwrap_or(true) { - return false; - } - if let Some(expire) = self.expire { - let now = proxmox::tools::time::epoch_i64(); - if expire > 0 && expire <= now { - return false; - } - } - true - } -} - -#[api( - properties: { - userid: { - type: Userid, - }, - comment: { - optional: true, - schema: SINGLE_LINE_COMMENT_SCHEMA, - }, - enable: { - optional: true, - schema: ENABLE_USER_SCHEMA, - }, - expire: { - optional: true, - schema: EXPIRE_USER_SCHEMA, - }, - firstname: { - optional: true, - schema: FIRST_NAME_SCHEMA, - }, - lastname: { - schema: LAST_NAME_SCHEMA, - optional: true, - }, - email: { - schema: EMAIL_SCHEMA, - optional: true, - }, - } -)] -#[derive(Serialize,Deserialize)] -/// User properties. -pub struct User { - pub userid: Userid, - #[serde(skip_serializing_if="Option::is_none")] - pub comment: Option, - #[serde(skip_serializing_if="Option::is_none")] - pub enable: Option, - #[serde(skip_serializing_if="Option::is_none")] - pub expire: Option, - #[serde(skip_serializing_if="Option::is_none")] - pub firstname: Option, - #[serde(skip_serializing_if="Option::is_none")] - pub lastname: Option, - #[serde(skip_serializing_if="Option::is_none")] - pub email: Option, -} - -impl User { - - pub fn is_active(&self) -> bool { - if !self.enable.unwrap_or(true) { - return false; - } - if let Some(expire) = self.expire { - let now = proxmox::tools::time::epoch_i64(); - if expire > 0 && expire <= now { - return false; - } - } - true - } -} - fn init() -> SectionConfig { let mut config = SectionConfig::new(&Authid::API_SCHEMA); diff --git a/src/lib.rs b/src/lib.rs index 4815c414..fcbc2e18 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,8 +9,6 @@ pub mod tools; #[macro_use] pub mod server; -pub mod pxar; - #[macro_use] pub mod backup; @@ -18,8 +16,6 @@ pub mod config; pub mod api2; -pub mod client; - pub mod auth_helpers; pub mod auth; diff --git a/src/server/pull.rs b/src/server/pull.rs index 9b454341..5214a218 100644 --- a/src/server/pull.rs +++ b/src/server/pull.rs @@ -22,10 +22,10 @@ use pbs_datastore::manifest::{ CLIENT_LOG_BLOB_NAME, MANIFEST_BLOB_NAME, ArchiveType, BackupManifest, FileInfo, archive_type }; use pbs_tools::sha::sha256; +use pbs_client::{BackupReader, BackupRepository, HttpClient, HttpClientOptions, RemoteChunkReader}; use crate::{ backup::DataStore, - client::{BackupReader, BackupRepository, HttpClient, HttpClientOptions, RemoteChunkReader}, server::WorkerTask, tools::ParallelHandler, }; diff --git a/src/server/rest.rs b/src/server/rest.rs index 3a359ad0..16680484 100644 --- a/src/server/rest.rs +++ b/src/server/rest.rs @@ -30,6 +30,8 @@ use proxmox::api::{ }; use proxmox::http_err; +use pbs_tools::compression::{DeflateEncoder, Level}; + use super::auth::AuthError; use super::environment::RestEnvironment; use super::formatter::*; @@ -39,7 +41,7 @@ use crate::api2::types::{Authid, Userid}; use crate::auth_helpers::*; use crate::config::cached_user_info::CachedUserInfo; use crate::tools; -use crate::tools::compression::{CompressionMethod, DeflateEncoder, Level}; +use crate::tools::compression::CompressionMethod; use crate::tools::AsyncReaderStream; use crate::tools::FileLogger; diff --git a/src/tools/compression.rs b/src/tools/compression.rs index b27d7e70..19626efc 100644 --- a/src/tools/compression.rs +++ b/src/tools/compression.rs @@ -1,19 +1,5 @@ -use std::io; -use std::pin::Pin; -use std::task::{Context, Poll}; - use anyhow::{bail, Error}; -use bytes::Bytes; -use flate2::{Compress, Compression, FlushCompress}; -use futures::ready; -use futures::stream::Stream; use hyper::header; -use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; - -use proxmox::io_format_err; -use proxmox::tools::byte_buffer::ByteBuffer; - -const BUFFER_SIZE: usize = 8192; /// Possible Compression Methods, order determines preference (later is preferred) #[derive(Eq, Ord, PartialEq, PartialOrd, Debug)] @@ -51,182 +37,3 @@ impl std::str::FromStr for CompressionMethod { } } } - -pub enum Level { - Fastest, - Best, - Default, - Precise(u32), -} - -#[derive(Eq, PartialEq)] -enum EncoderState { - Reading, - Writing, - Flushing, - Finished, -} - -pub struct DeflateEncoder { - inner: T, - compressor: Compress, - buffer: ByteBuffer, - input_buffer: Bytes, - state: EncoderState, -} - -impl DeflateEncoder { - pub fn new(inner: T) -> Self { - Self::with_quality(inner, Level::Default) - } - - pub fn with_quality(inner: T, level: Level) -> Self { - let level = match level { - Level::Fastest => Compression::fast(), - Level::Best => Compression::best(), - Level::Default => Compression::new(3), - Level::Precise(val) => Compression::new(val), - }; - - Self { - inner, - compressor: Compress::new(level, false), - buffer: ByteBuffer::with_capacity(BUFFER_SIZE), - input_buffer: Bytes::new(), - state: EncoderState::Reading, - } - } - - pub fn total_in(&self) -> u64 { - self.compressor.total_in() - } - - pub fn total_out(&self) -> u64 { - self.compressor.total_out() - } - - pub fn into_inner(self) -> T { - self.inner - } - - fn encode( - &mut self, - inbuf: &[u8], - flush: FlushCompress, - ) -> Result<(usize, flate2::Status), io::Error> { - let old_in = self.compressor.total_in(); - let old_out = self.compressor.total_out(); - let res = self - .compressor - .compress(&inbuf[..], self.buffer.get_free_mut_slice(), flush)?; - let new_in = (self.compressor.total_in() - old_in) as usize; - let new_out = (self.compressor.total_out() - old_out) as usize; - self.buffer.add_size(new_out); - - Ok((new_in, res)) - } -} - -impl DeflateEncoder> { - // assume small files - pub async fn compress_vec(&mut self, reader: &mut R, size_hint: usize) -> Result<(), Error> - where - R: AsyncRead + Unpin, - { - let mut buffer = Vec::with_capacity(size_hint); - reader.read_to_end(&mut buffer).await?; - self.inner.reserve(size_hint); // should be enough since we want smalller files - self.compressor.compress_vec(&buffer[..], &mut self.inner, FlushCompress::Finish)?; - Ok(()) - } -} - -impl DeflateEncoder { - pub async fn compress(&mut self, reader: &mut R) -> Result<(), Error> - where - R: AsyncRead + Unpin, - { - let mut buffer = ByteBuffer::with_capacity(BUFFER_SIZE); - let mut eof = false; - loop { - if !eof && !buffer.is_full() { - let read = buffer.read_from_async(reader).await?; - if read == 0 { - eof = true; - } - } - let (read, _res) = self.encode(&buffer[..], FlushCompress::None)?; - buffer.consume(read); - - self.inner.write_all(&self.buffer[..]).await?; - self.buffer.clear(); - - if buffer.is_empty() && eof { - break; - } - } - - loop { - let (_read, res) = self.encode(&[][..], FlushCompress::Finish)?; - self.inner.write_all(&self.buffer[..]).await?; - self.buffer.clear(); - if res == flate2::Status::StreamEnd { - break; - } - } - - Ok(()) - } -} - -impl Stream for DeflateEncoder -where - T: Stream> + Unpin, - O: Into -{ - type Item = Result; - - fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - let this = self.get_mut(); - - loop { - match this.state { - EncoderState::Reading => { - if let Some(res) = ready!(Pin::new(&mut this.inner).poll_next(cx)) { - let buf = res?; - this.input_buffer = buf.into(); - this.state = EncoderState::Writing; - } else { - this.state = EncoderState::Flushing; - } - } - EncoderState::Writing => { - if this.input_buffer.is_empty() { - return Poll::Ready(Some(Err(io_format_err!("empty input during write")))); - } - let mut buf = this.input_buffer.split_off(0); - let (read, res) = this.encode(&buf[..], FlushCompress::None)?; - this.input_buffer = buf.split_off(read); - if this.input_buffer.is_empty() { - this.state = EncoderState::Reading; - } - if this.buffer.is_full() || res == flate2::Status::BufError { - let bytes = this.buffer.remove_data(this.buffer.len()).to_vec(); - return Poll::Ready(Some(Ok(bytes.into()))); - } - } - EncoderState::Flushing => { - let (_read, res) = this.encode(&[][..], FlushCompress::Finish)?; - if !this.buffer.is_empty() { - let bytes = this.buffer.remove_data(this.buffer.len()).to_vec(); - return Poll::Ready(Some(Ok(bytes.into()))); - } - if res == flate2::Status::StreamEnd { - this.state = EncoderState::Finished; - } - } - EncoderState::Finished => return Poll::Ready(None), - } - } - } -} diff --git a/src/tools/mod.rs b/src/tools/mod.rs index 658c7014..d13c4d45 100644 --- a/src/tools/mod.rs +++ b/src/tools/mod.rs @@ -2,8 +2,6 @@ //! //! This is a collection of small and useful tools. use std::any::Any; -use std::collections::HashMap; -use std::hash::BuildHasher; use std::fs::File; use std::io::{self, BufRead}; use std::os::unix::io::RawFd; @@ -29,7 +27,6 @@ pub use pbs_tools::process_locker::{ ProcessLocker, ProcessLockExclusiveGuard, ProcessLockSharedGuard }; -pub mod acl; pub mod apt; pub mod async_io; pub mod compression; @@ -51,8 +48,6 @@ pub mod statistics; pub mod subscription; pub mod systemd; pub mod ticket; -pub mod xattr; -pub mod zip; pub mod sgutils2; pub mod paperkey; @@ -69,6 +64,7 @@ mod file_logger; pub use file_logger::{FileLogger, FileLogOptions}; pub use pbs_tools::broadcast_future::{BroadcastData, BroadcastFuture}; +pub use pbs_tools::ops::ControlFlow; /// The `BufferedRead` trait provides a single function /// `buffered_read`. It returns a reference to an internal buffer. The @@ -122,65 +118,6 @@ pub fn required_array_property<'a>(param: &'a Value, name: &str) -> Result<&'a [ } } -pub fn complete_file_name(arg: &str, _param: &HashMap) -> Vec -where - S: BuildHasher, -{ - let mut result = vec![]; - - use nix::fcntl::AtFlags; - use nix::fcntl::OFlag; - use nix::sys::stat::Mode; - - let mut dirname = std::path::PathBuf::from(if arg.is_empty() { "./" } else { arg }); - - let is_dir = match nix::sys::stat::fstatat(libc::AT_FDCWD, &dirname, AtFlags::empty()) { - Ok(stat) => (stat.st_mode & libc::S_IFMT) == libc::S_IFDIR, - Err(_) => false, - }; - - if !is_dir { - if let Some(parent) = dirname.parent() { - dirname = parent.to_owned(); - } - } - - let mut dir = - match nix::dir::Dir::openat(libc::AT_FDCWD, &dirname, OFlag::O_DIRECTORY, Mode::empty()) { - Ok(d) => d, - Err(_) => return result, - }; - - for item in dir.iter() { - if let Ok(entry) = item { - if let Ok(name) = entry.file_name().to_str() { - if name == "." || name == ".." { - continue; - } - let mut newpath = dirname.clone(); - newpath.push(name); - - if let Ok(stat) = - nix::sys::stat::fstatat(libc::AT_FDCWD, &newpath, AtFlags::empty()) - { - if (stat.st_mode & libc::S_IFMT) == libc::S_IFDIR { - newpath.push(""); - if let Some(newpath) = newpath.to_str() { - result.push(newpath.to_owned()); - } - continue; - } - } - if let Some(newpath) = newpath.to_str() { - result.push(newpath.to_owned()); - } - } - } - } - - result -} - /// Shortcut for md5 sums. pub fn md5sum(data: &[u8]) -> Result { hash(MessageDigest::md5(), data).map_err(Error::from) @@ -373,17 +310,6 @@ pub fn setup_safe_path_env() { } } -pub fn strip_ascii_whitespace(line: &[u8]) -> &[u8] { - let line = match line.iter().position(|&b| !b.is_ascii_whitespace()) { - Some(n) => &line[n..], - None => return &[], - }; - match line.iter().rev().position(|&b| !b.is_ascii_whitespace()) { - Some(n) => &line[..(line.len() - n)], - None => &[], - } -} - /// Create the base run-directory. /// /// This exists to fixate the permissions for the run *base* directory while allowing intermediate @@ -396,14 +322,3 @@ pub fn create_run_dir() -> Result<(), Error> { let _: bool = create_path(pbs_buildcfg::PROXMOX_BACKUP_RUN_DIR_M!(), None, Some(opts))?; Ok(()) } - -/// Modeled after the nightly `std::ops::ControlFlow`. -#[derive(Clone, Copy, Debug, PartialEq)] -pub enum ControlFlow { - Continue(C), - Break(B), -} - -impl ControlFlow { - pub const CONTINUE: ControlFlow = ControlFlow::Continue(()); -} diff --git a/tests/catar.rs b/tests/catar.rs index 550600c6..0eea3945 100644 --- a/tests/catar.rs +++ b/tests/catar.rs @@ -1,7 +1,8 @@ +use std::process::Command; + use anyhow::{Error}; -use std::process::Command; -use proxmox_backup::pxar::*; +use pbs_client::pxar::*; fn run_test(dir_name: &str) -> Result<(), Error> { -- 2.39.2