src/api2/reader.rs

   1 //use chrono::{Local, TimeZone};
   2 use anyhow::{bail, format_err, Error};
   3 use futures::*;
   4 use hyper::header::{self, HeaderValue, UPGRADE};
   5 use hyper::http::request::Parts;
   6 use hyper::{Body, Response, StatusCode};
   7 use serde_json::Value;
   8
   9 use proxmox::{sortable, identity};
  10 use proxmox::api::{ApiResponseFuture, ApiHandler, ApiMethod, Router, RpcEnvironment, Permission};
  11 use proxmox::api::schema::*;
  12 use proxmox::http_err;
  13
  14 use crate::api2::types::*;
  15 use crate::backup::*;
  16 use crate::server::{WorkerTask, H2Service};
  17 use crate::tools;
  18 use crate::config::acl::PRIV_DATASTORE_READ;
  19 use crate::config::cached_user_info::CachedUserInfo;
  20 use crate::api2::helpers;
  21
  22 mod environment;
  23 use environment::*;
  24
  25 pub const ROUTER: Router = Router::new()
  26     .upgrade(&API_METHOD_UPGRADE_BACKUP);
  27
  28 #[sortable]
  29 pub const API_METHOD_UPGRADE_BACKUP: ApiMethod = ApiMethod::new(
  30     &ApiHandler::AsyncHttp(&upgrade_to_backup_reader_protocol),
  31     &ObjectSchema::new(
  32         concat!("Upgraded to backup protocol ('", PROXMOX_BACKUP_READER_PROTOCOL_ID_V1!(), "')."),
  33         &sorted!([
  34             ("store", false, &DATASTORE_SCHEMA),
  35             ("backup-type", false, &BACKUP_TYPE_SCHEMA),
  36             ("backup-id", false, &BACKUP_ID_SCHEMA),
  37             ("backup-time", false, &BACKUP_TIME_SCHEMA),
  38             ("debug", true, &BooleanSchema::new("Enable verbose debug logging.").schema()),
  39         ]),
  40     )
  41 ).access(
  42     // Note: parameter 'store' is no uri parameter, so we need to test inside function body
  43     Some("The user needs Datastore.Read privilege on /datastore/{store}."),
  44     &Permission::Anybody
  45 );
  46
  47 fn upgrade_to_backup_reader_protocol(
  48     parts: Parts,
  49     req_body: Body,
  50     param: Value,
  51     _info: &ApiMethod,
  52     rpcenv: Box<dyn RpcEnvironment>,
  53 ) -> ApiResponseFuture {
  54
  55     async move {
  56         let debug = param["debug"].as_bool().unwrap_or(false);
  57
  58         let username = rpcenv.get_user().unwrap();
  59         let store = tools::required_string_param(&param, "store")?.to_owned();
  60
  61         let user_info = CachedUserInfo::new()?;
  62         user_info.check_privs(&username, &["datastore", &store], PRIV_DATASTORE_READ, false)?;
  63
  64         let datastore = DataStore::lookup_datastore(&store)?;
  65
  66         let backup_type = tools::required_string_param(&param, "backup-type")?;
  67         let backup_id = tools::required_string_param(&param, "backup-id")?;
  68         let backup_time = tools::required_integer_param(&param, "backup-time")?;
  69
  70         let protocols = parts
  71             .headers
  72             .get("UPGRADE")
  73             .ok_or_else(|| format_err!("missing Upgrade header"))?
  74         .to_str()?;
  75
  76         if protocols != PROXMOX_BACKUP_READER_PROTOCOL_ID_V1!() {
  77             bail!("invalid protocol name");
  78         }
  79
  80         if parts.version >=  http::version::Version::HTTP_2 {
  81             bail!("unexpected http version '{:?}' (expected version < 2)", parts.version);
  82         }
  83
  84         let env_type = rpcenv.env_type();
  85
  86         let backup_dir = BackupDir::new(backup_type, backup_id, backup_time);
  87         let path = datastore.base_path();
  88
  89         //let files = BackupInfo::list_files(&path, &backup_dir)?;
  90
  91         let worker_id = format!("{}_{}_{}_{:08X}", store, backup_type, backup_id, backup_dir.backup_time().timestamp());
  92
  93         WorkerTask::spawn("reader", Some(worker_id), &username.clone(), true, move |worker| {
  94             let mut env = ReaderEnvironment::new(
  95                 env_type, username.clone(), worker.clone(), datastore, backup_dir);
  96
  97             env.debug = debug;
  98
  99             env.log(format!("starting new backup reader datastore '{}': {:?}", store, path));
 100
 101             let service = H2Service::new(env.clone(), worker.clone(), &READER_API_ROUTER, debug);
 102
 103             let abort_future = worker.abort_future();
 104
 105             let req_fut = req_body
 106                 .on_upgrade()
 107                 .map_err(Error::from)
 108                 .and_then({
 109                     let env = env.clone();
 110                     move |conn| {
 111                         env.debug("protocol upgrade done");
 112
 113                         let mut http = hyper::server::conn::Http::new();
 114                         http.http2_only(true);
 115                         // increase window size: todo - find optiomal size
 116                         let window_size = 32*1024*1024; // max = (1 << 31) - 2
 117                         http.http2_initial_stream_window_size(window_size);
 118                         http.http2_initial_connection_window_size(window_size);
 119
 120                         http.serve_connection(conn, service)
 121                             .map_err(Error::from)
 122                     }
 123                 });
 124             let abort_future = abort_future
 125                 .map(|_| Err(format_err!("task aborted")));
 126
 127             use futures::future::Either;
 128             futures::future::select(req_fut, abort_future)
 129                 .map(|res| match res {
 130                     Either::Left((Ok(res), _)) => Ok(res),
 131                     Either::Left((Err(err), _)) => Err(err),
 132                     Either::Right((Ok(res), _)) => Ok(res),
 133                     Either::Right((Err(err), _)) => Err(err),
 134                 })
 135                 .map_ok(move |_| env.log("reader finished successfully"))
 136         })?;
 137
 138         let response = Response::builder()
 139             .status(StatusCode::SWITCHING_PROTOCOLS)
 140             .header(UPGRADE, HeaderValue::from_static(PROXMOX_BACKUP_READER_PROTOCOL_ID_V1!()))
 141             .body(Body::empty())?;
 142
 143         Ok(response)
 144     }.boxed()
 145 }
 146
 147 pub const READER_API_ROUTER: Router = Router::new()
 148     .subdirs(&[
 149         (
 150             "chunk", &Router::new()
 151                 .download(&API_METHOD_DOWNLOAD_CHUNK)
 152         ),
 153         (
 154             "download", &Router::new()
 155                 .download(&API_METHOD_DOWNLOAD_FILE)
 156         ),
 157         (
 158             "speedtest", &Router::new()
 159                 .download(&API_METHOD_SPEEDTEST)
 160         ),
 161     ]);
 162
 163 #[sortable]
 164 pub const API_METHOD_DOWNLOAD_FILE: ApiMethod = ApiMethod::new(
 165     &ApiHandler::AsyncHttp(&download_file),
 166     &ObjectSchema::new(
 167         "Download specified file.",
 168         &sorted!([
 169             ("file-name", false, &crate::api2::types::BACKUP_ARCHIVE_NAME_SCHEMA),
 170         ]),
 171     )
 172 );
 173
 174 fn download_file(
 175     _parts: Parts,
 176     _req_body: Body,
 177     param: Value,
 178     _info: &ApiMethod,
 179     rpcenv: Box<dyn RpcEnvironment>,
 180 ) -> ApiResponseFuture {
 181
 182     async move {
 183         let env: &ReaderEnvironment = rpcenv.as_ref();
 184
 185         let file_name = tools::required_string_param(&param, "file-name")?.to_owned();
 186
 187         let mut path = env.datastore.base_path();
 188         path.push(env.backup_dir.relative_path());
 189         path.push(&file_name);
 190
 191         env.log(format!("download {:?}", path.clone()));
 192
 193         helpers::create_download_response(path).await
 194     }.boxed()
 195 }
 196
 197 #[sortable]
 198 pub const API_METHOD_DOWNLOAD_CHUNK: ApiMethod = ApiMethod::new(
 199     &ApiHandler::AsyncHttp(&download_chunk),
 200     &ObjectSchema::new(
 201         "Download specified chunk.",
 202         &sorted!([
 203             ("digest", false, &CHUNK_DIGEST_SCHEMA),
 204         ]),
 205     )
 206 );
 207
 208 fn download_chunk(
 209     _parts: Parts,
 210     _req_body: Body,
 211     param: Value,
 212     _info: &ApiMethod,
 213     rpcenv: Box<dyn RpcEnvironment>,
 214 ) -> ApiResponseFuture {
 215
 216     async move {
 217         let env: &ReaderEnvironment = rpcenv.as_ref();
 218
 219         let digest_str = tools::required_string_param(&param, "digest")?;
 220         let digest = proxmox::tools::hex_to_digest(digest_str)?;
 221
 222         let (path, _) = env.datastore.chunk_path(&digest);
 223         let path2 = path.clone();
 224
 225         env.debug(format!("download chunk {:?}", path));
 226
 227         let data = tokio::fs::read(path)
 228             .map_err(move |err| http_err!(BAD_REQUEST, format!("reading file {:?} failed: {}", path2, err)))
 229             .await?;
 230
 231         let body = Body::from(data);
 232
 233         // fixme: set other headers ?
 234         Ok(Response::builder()
 235            .status(StatusCode::OK)
 236            .header(header::CONTENT_TYPE, "application/octet-stream")
 237            .body(body)
 238            .unwrap())
 239     }.boxed()
 240 }
 241
 242 /* this is too slow
 243 fn download_chunk_old(
 244     _parts: Parts,
 245     _req_body: Body,
 246     param: Value,
 247     _info: &ApiMethod,
 248     rpcenv: Box<dyn RpcEnvironment>,
 249 ) -> Result<ApiResponseFuture, Error> {
 250
 251     let env: &ReaderEnvironment = rpcenv.as_ref();
 252     let env2 = env.clone();
 253
 254     let digest_str = tools::required_string_param(&param, "digest")?;
 255     let digest = proxmox::tools::hex_to_digest(digest_str)?;
 256
 257     let (path, _) = env.datastore.chunk_path(&digest);
 258
 259     let path2 = path.clone();
 260     let path3 = path.clone();
 261
 262     let response_future = tokio::fs::File::open(path)
 263         .map_err(move |err| http_err!(BAD_REQUEST, format!("open file {:?} failed: {}", path2, err)))
 264         .and_then(move |file| {
 265             env2.debug(format!("download chunk {:?}", path3));
 266             let payload = tokio_util::codec::FramedRead::new(file, tokio_util::codec::BytesCodec::new())
 267                 .map_ok(|bytes| hyper::body::Bytes::from(bytes.freeze()));
 268
 269             let body = Body::wrap_stream(payload);
 270
 271             // fixme: set other headers ?
 272             futures::future::ok(Response::builder()
 273                .status(StatusCode::OK)
 274                .header(header::CONTENT_TYPE, "application/octet-stream")
 275                .body(body)
 276                .unwrap())
 277         });
 278
 279     Ok(Box::new(response_future))
 280 }
 281 */
 282
 283 pub const API_METHOD_SPEEDTEST: ApiMethod = ApiMethod::new(
 284     &ApiHandler::AsyncHttp(&speedtest),
 285     &ObjectSchema::new("Test 4M block download speed.", &[])
 286 );
 287
 288 fn speedtest(
 289     _parts: Parts,
 290     _req_body: Body,
 291     _param: Value,
 292     _info: &ApiMethod,
 293     _rpcenv: Box<dyn RpcEnvironment>,
 294 ) -> ApiResponseFuture {
 295
 296     let buffer = vec![65u8; 1024*1024]; // nonsense [A,A,A...]
 297
 298     let body = Body::from(buffer);
 299
 300     let response = Response::builder()
 301         .status(StatusCode::OK)
 302         .header(header::CONTENT_TYPE, "application/octet-stream")
 303         .body(body)
 304         .unwrap();
 305
 306     future::ok(response).boxed()
 307 }