1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab ft=cpp
5 * Ceph - scalable distributed file system
7 * Copyright (C) 2019 Red Hat, Inc.
9 * This is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU Lesser General Public
11 * License version 2.1, as published by the Free Software
12 * Foundation. See file COPYING.
19 #include "rgw_notify_event_type.h"
20 #include "common/tracer.h"
24 class RGWAccessListFilter
;
31 struct rgw_user_bucket
;
33 class RGWCoroutinesManagerRegistry
;
34 class RGWListRawObjsCtx
;
35 class RGWBucketSyncPolicyHandler
;
36 using RGWBucketSyncPolicyHandlerRef
= std::shared_ptr
<RGWBucketSyncPolicyHandler
>;
37 class RGWDataSyncStatusManager
;
38 class RGWSyncModuleInstance
;
39 typedef std::shared_ptr
<RGWSyncModuleInstance
> RGWSyncModuleInstanceRef
;
40 class RGWCompressionInfo
;
43 using RGWBucketListNameFilter
= std::function
<bool (const std::string
&)>;
48 namespace IAM
{ struct Policy
; }
53 virtual int handle_data(bufferlist
& bl
, off_t bl_ofs
, off_t bl_len
) = 0;
55 virtual ~RGWGetDataCB() {}
59 std::string read_iter
;
62 RGWUsageIter() : index(0) {}
66 * @struct RGWClusterStat
67 * Cluster-wide usage information
69 struct RGWClusterStat
{
74 /// total available/free
80 class RGWGetBucketStats_CB
: public RefCountedObject
{
83 std::map
<RGWObjCategory
, RGWStorageStats
>* stats
;
85 explicit RGWGetBucketStats_CB(const rgw_bucket
& _bucket
) : bucket(_bucket
), stats(NULL
) {}
86 ~RGWGetBucketStats_CB() override
{}
87 virtual void handle_response(int r
) = 0;
88 virtual void set_response(std::map
<RGWObjCategory
, RGWStorageStats
>* _stats
) {
93 class RGWGetUserStats_CB
: public RefCountedObject
{
96 RGWStorageStats stats
;
98 explicit RGWGetUserStats_CB(const rgw_user
& _user
) : user(_user
) {}
99 ~RGWGetUserStats_CB() override
{}
100 virtual void handle_response(int r
) = 0;
101 virtual void set_response(RGWStorageStats
& _stats
) {
107 * @defgroup RGWSAL RGW Store Abstraction Layer
109 * The Store Abstraction Layer is an API that separates the top layer of RGW that
110 * handles client protocols (such as S3 or Swift) from the bottom layer of RGW that
111 * interacts with a backing store. It allows the creation of multiple backing stores
112 * that can co-exist with a single RGW instance, and allows the creation of stacking
113 * layers of translators that can modify operations as they pass down the stack.
114 * Examples of translators might be a cache layer, a duplication layer that copies
115 * operations to multiple stores, or a policy layer that sends some operations to one
116 * store and some to another.
118 * The basic unit of a SAL implementation is the Store. Whether an actual backing store
119 * or a translator, there will be a Store implementation that represents it. Examples
120 * are the RadosStore that communicates via RADOS with a Ceph cluster, and the DBStore
121 * that uses a SQL db (such as SQLite3) as a backing store. There is a singleton
122 * instance of each Store.
124 * Data within RGW is owned by a User. The User is the unit of authentication and
127 * Data within RGW is stored as an Object. Each Object is a single chunk of data, owned
128 * by a single User, contained within a single Bucket. It has metadata associated with
129 * it, such as size, owner, and so in, and a set of key-value attributes that can
130 * contain anything needed by the top half.
132 * Data with RGW is organized into Buckets. Each Bucket is owned by a User, and
133 * contains Objects. There is a single, flat layer of Buckets, there is no hierarchy,
134 * and each Object is contained in a single Bucket.
136 * Instantiations of SAL classes are done as unique pointers, using std::unique_ptr.
137 * Instances of these classes are acquired via getters, and it's up to the caller to
138 * manage the lifetime.
140 * @note Anything using RGWObjContext is subject to change, as that type will not be
141 * used in the final API.
147 * @brief Base abstractions and API for SAL
150 namespace rgw
{ namespace sal
{
157 #define RGW_SAL_VERSION 1
163 class MultipartUpload
;
170 class LuaScriptManager
;
171 class RGWOIDCProvider
;
176 ATTRSMOD_REPLACE
= 1,
180 // a simple streaming data processing abstraction
182 * @brief A simple streaming data processing abstraction
184 class DataProcessor
{
186 virtual ~DataProcessor() {}
189 * @brief Consume a bufferlist in its entirety at the given object offset.
191 * An empty bufferlist is given to request that any buffered data be flushed, though this doesn't
192 * wait for completions
194 virtual int process(bufferlist
&& data
, uint64_t offset
) = 0;
198 * @brief a data consumer that writes an object in a bucket
200 class ObjectProcessor
: public DataProcessor
{
202 /** prepare to start processing object data */
203 virtual int prepare(optional_yield y
) = 0;
205 /** complete the operation and make its result visible to clients */
206 virtual int complete(size_t accounted_size
, const std::string
& etag
,
207 ceph::real_time
*mtime
, ceph::real_time set_mtime
,
208 std::map
<std::string
, bufferlist
>& attrs
,
209 ceph::real_time delete_at
,
210 const char *if_match
, const char *if_nomatch
,
211 const std::string
*user_data
,
212 rgw_zone_set
*zones_trace
, bool *canceled
,
213 optional_yield y
) = 0;
216 /** Base class for AIO completions */
220 virtual ~Completions() = default;
221 virtual int drain() = 0;
224 /** A list of key-value attributes */
225 using Attrs
= std::map
<std::string
, ceph::buffer::list
>;
228 * @brief Base singleton representing a Store or stacking layer
230 * The Store is the base abstraction of the SAL layer. It represents a base storage
231 * mechanism, or a intermediate stacking layer. There is a single instance of a given
232 * Store per RGW, and this Store mediates all access to it's backing.
234 * A store contains, loosely, @a User, @a Bucket, and @a Object entities. The @a Object
235 * contains data, and it's associated metadata. The @a Bucket contains Objects, and
236 * metadata about the bucket. Both Buckets and Objects are owned by a @a User, which is
237 * the basic unit of access control.
239 * A store also has metadata and some global responsibilities. For example, a store is
240 * responsible for managing the LifeCycle activities for it's data.
245 virtual ~Store() = default;
247 /** Name of this store provider (e.g., RADOS") */
248 virtual const char* get_name() const = 0;
249 /** Get cluster unique identifier */
250 virtual std::string
get_cluster_id(const DoutPrefixProvider
* dpp
, optional_yield y
) = 0;
251 /** Get a User from a rgw_user. Does not query store for user info, so quick */
252 virtual std::unique_ptr
<User
> get_user(const rgw_user
& u
) = 0;
253 /** Lookup a User by access key. Queries store for user info. */
254 virtual int get_user_by_access_key(const DoutPrefixProvider
* dpp
, const std::string
& key
, optional_yield y
, std::unique_ptr
<User
>* user
) = 0;
255 /** Lookup a User by email address. Queries store for user info. */
256 virtual int get_user_by_email(const DoutPrefixProvider
* dpp
, const std::string
& email
, optional_yield y
, std::unique_ptr
<User
>* user
) = 0;
257 /** Lookup a User by swift username. Queries store for user info. */
258 virtual int get_user_by_swift(const DoutPrefixProvider
* dpp
, const std::string
& user_str
, optional_yield y
, std::unique_ptr
<User
>* user
) = 0;
259 /** Get a basic Object. This Object is not looked up, and is incomplete, since is
260 * does not have a bucket. This should only be used when an Object is needed before
261 * there is a Bucket, otherwise use the get_object() in the Bucket class. */
262 virtual std::unique_ptr
<Object
> get_object(const rgw_obj_key
& k
) = 0;
263 /** Get a Bucket by info. Does not query the store, just uses the give bucket info. */
264 virtual int get_bucket(User
* u
, const RGWBucketInfo
& i
, std::unique_ptr
<Bucket
>* bucket
) = 0;
265 /** Lookup a Bucket by key. Queries store for bucket info. */
266 virtual int get_bucket(const DoutPrefixProvider
* dpp
, User
* u
, const rgw_bucket
& b
, std::unique_ptr
<Bucket
>* bucket
, optional_yield y
) = 0;
267 /** Lookup a Bucket by name. Queries store for bucket info. */
268 virtual int get_bucket(const DoutPrefixProvider
* dpp
, User
* u
, const std::string
& tenant
, const std::string
& name
, std::unique_ptr
<Bucket
>* bucket
, optional_yield y
) = 0;
269 /** For multisite, this Store is the zone's master */
270 virtual bool is_meta_master() = 0;
271 /** For multisite, forward an OP to the zone's master */
272 virtual int forward_request_to_master(const DoutPrefixProvider
*dpp
, User
* user
, obj_version
* objv
,
273 bufferlist
& in_data
, JSONParser
* jp
, req_info
& info
,
274 optional_yield y
) = 0;
275 /** Get zone info for this store */
276 virtual Zone
* get_zone() = 0;
277 /** Get a unique ID specific to this zone. */
278 virtual std::string
zone_unique_id(uint64_t unique_num
) = 0;
279 /** Get a unique Swift transaction ID specific to this zone */
280 virtual std::string
zone_unique_trans_id(const uint64_t unique_num
) = 0;
281 /** Get statistics about the cluster represented by this Store */
282 virtual int cluster_stat(RGWClusterStat
& stats
) = 0;
283 /** Get a @a Lifecycle object. Used to manage/run lifecycle transitions */
284 virtual std::unique_ptr
<Lifecycle
> get_lifecycle(void) = 0;
285 /** Get a @a Completions object. Used for Async I/O tracking */
286 virtual std::unique_ptr
<Completions
> get_completions(void) = 0;
288 /** Get a @a Notification object. Used to communicate with non-RGW daemons, such as
289 * management/tracking software */
291 virtual std::unique_ptr
<Notification
> get_notification(rgw::sal::Object
* obj
, rgw::sal::Object
* src_obj
, struct req_state
* s
,
292 rgw::notify::EventType event_type
, const std::string
* object_name
=nullptr) = 0;
293 /** No-req_state variant (e.g., rgwlc) */
294 virtual std::unique_ptr
<Notification
> get_notification(
295 const DoutPrefixProvider
* dpp
, rgw::sal::Object
* obj
, rgw::sal::Object
* src_obj
, RGWObjectCtx
* rctx
,
296 rgw::notify::EventType event_type
, rgw::sal::Bucket
* _bucket
, std::string
& _user_id
, std::string
& _user_tenant
,
297 std::string
& _req_id
, optional_yield y
) = 0;
299 /** Get access to the lifecycle management thread */
300 virtual RGWLC
* get_rgwlc(void) = 0;
301 /** Get access to the coroutine registry. Used to create new coroutine managers */
302 virtual RGWCoroutinesManagerRegistry
* get_cr_registry() = 0;
304 /** Log usage data to the store. Usage data is things like bytes sent/received and
306 virtual int log_usage(const DoutPrefixProvider
*dpp
, std::map
<rgw_user_bucket
, RGWUsageBatch
>& usage_info
) = 0;
307 /** Log OP data to the store. Data is opaque to SAL */
308 virtual int log_op(const DoutPrefixProvider
*dpp
, std::string
& oid
, bufferlist
& bl
) = 0;
309 /** Register this Store to the service map. Somewhat Rados specific; may be removed*/
310 virtual int register_to_service_map(const DoutPrefixProvider
*dpp
, const std::string
& daemon_type
,
311 const std::map
<std::string
, std::string
>& meta
) = 0;
312 /** Get default quota info. Used as fallback if a user or bucket has no quota set*/
313 virtual void get_quota(RGWQuotaInfo
& bucket_quota
, RGWQuotaInfo
& user_quota
) = 0;
314 /** Get global rate limit configuration*/
315 virtual void get_ratelimit(RGWRateLimitInfo
& bucket_ratelimit
, RGWRateLimitInfo
& user_ratelimit
, RGWRateLimitInfo
& anon_ratelimit
) = 0;
316 /** Enable or disable a set of bucket. e.g. if a User is suspended */
317 virtual int set_buckets_enabled(const DoutPrefixProvider
* dpp
, std::vector
<rgw_bucket
>& buckets
, bool enabled
) = 0;
318 /** Get a new request ID */
319 virtual uint64_t get_new_req_id() = 0;
320 /** Get a handler for bucket sync policy. */
321 virtual int get_sync_policy_handler(const DoutPrefixProvider
* dpp
,
322 std::optional
<rgw_zone_id
> zone
,
323 std::optional
<rgw_bucket
> bucket
,
324 RGWBucketSyncPolicyHandlerRef
* phandler
,
325 optional_yield y
) = 0;
326 /** Get a status manager for bucket sync */
327 virtual RGWDataSyncStatusManager
* get_data_sync_manager(const rgw_zone_id
& source_zone
) = 0;
328 /** Wake up sync threads for bucket metadata sync */
329 virtual void wakeup_meta_sync_shards(std::set
<int>& shard_ids
) = 0;
330 /** Wake up sync threads for bucket data sync */
331 virtual void wakeup_data_sync_shards(const DoutPrefixProvider
*dpp
, const rgw_zone_id
& source_zone
, std::map
<int, std::set
<std::string
> >& shard_ids
) = 0;
332 /** Clear all usage statistics globally */
333 virtual int clear_usage(const DoutPrefixProvider
*dpp
) = 0;
334 /** Get usage statistics for all users and buckets */
335 virtual int read_all_usage(const DoutPrefixProvider
*dpp
, uint64_t start_epoch
, uint64_t end_epoch
,
336 uint32_t max_entries
, bool* is_truncated
,
337 RGWUsageIter
& usage_iter
,
338 std::map
<rgw_user_bucket
, rgw_usage_log_entry
>& usage
) = 0;
339 /** Trim usage log for all users and buckets */
340 virtual int trim_all_usage(const DoutPrefixProvider
*dpp
, uint64_t start_epoch
, uint64_t end_epoch
) = 0;
341 /** Get a configuration value for the given name */
342 virtual int get_config_key_val(std::string name
, bufferlist
* bl
) = 0;
343 /** Start a metadata listing of the given section */
344 virtual int meta_list_keys_init(const DoutPrefixProvider
*dpp
, const std::string
& section
, const std::string
& marker
, void** phandle
) = 0;
345 /** Get the next key from a metadata list */
346 virtual int meta_list_keys_next(const DoutPrefixProvider
*dpp
, void* handle
, int max
, std::list
<std::string
>& keys
, bool* truncated
) = 0;
347 /** Complete a metadata listing */
348 virtual void meta_list_keys_complete(void* handle
) = 0;
349 /** Get the marker associated with the current metadata listing */
350 virtual std::string
meta_get_marker(void* handle
) = 0;
351 /** Remove a specific metadata key */
352 virtual int meta_remove(const DoutPrefixProvider
* dpp
, std::string
& metadata_key
, optional_yield y
) = 0;
353 /** Get an instance of the Sync module for bucket sync */
354 virtual const RGWSyncModuleInstanceRef
& get_sync_module() = 0;
355 /** Get the ID of the current host */
356 virtual std::string
get_host_id() = 0;
357 /** Get a Lua script manager for running lua scripts */
358 virtual std::unique_ptr
<LuaScriptManager
> get_lua_script_manager() = 0;
359 /** Get an IAM Role by name etc. */
360 virtual std::unique_ptr
<RGWRole
> get_role(std::string name
,
363 std::string trust_policy
="",
364 std::string max_session_duration_str
="",
365 std::multimap
<std::string
,std::string
> tags
={}) = 0;
366 /** Get an IAM Role by ID */
367 virtual std::unique_ptr
<RGWRole
> get_role(std::string id
) = 0;
368 /** Get all IAM Roles optionally filtered by path */
369 virtual int get_roles(const DoutPrefixProvider
*dpp
,
371 const std::string
& path_prefix
,
372 const std::string
& tenant
,
373 std::vector
<std::unique_ptr
<RGWRole
>>& roles
) = 0;
374 /** Get an empty Open ID Connector provider */
375 virtual std::unique_ptr
<RGWOIDCProvider
> get_oidc_provider() = 0;
376 /** Get all Open ID Connector providers, optionally filtered by tenant */
377 virtual int get_oidc_providers(const DoutPrefixProvider
*dpp
,
378 const std::string
& tenant
,
379 std::vector
<std::unique_ptr
<RGWOIDCProvider
>>& providers
) = 0;
380 /** Get a Writer that appends to an object */
381 virtual std::unique_ptr
<Writer
> get_append_writer(const DoutPrefixProvider
*dpp
,
383 std::unique_ptr
<rgw::sal::Object
> _head_obj
,
384 const rgw_user
& owner
, RGWObjectCtx
& obj_ctx
,
385 const rgw_placement_rule
*ptail_placement_rule
,
386 const std::string
& unique_tag
,
388 uint64_t *cur_accounted_size
) = 0;
389 /** Get a Writer that atomically writes an entire object */
390 virtual std::unique_ptr
<Writer
> get_atomic_writer(const DoutPrefixProvider
*dpp
,
392 std::unique_ptr
<rgw::sal::Object
> _head_obj
,
393 const rgw_user
& owner
, RGWObjectCtx
& obj_ctx
,
394 const rgw_placement_rule
*ptail_placement_rule
,
396 const std::string
& unique_tag
) = 0;
398 /** Clean up a store for termination */
399 virtual void finalize(void) = 0;
401 /** Get the Ceph context associated with this store. May be removed. */
402 virtual CephContext
* ctx(void) = 0;
404 /** Get the location of where lua packages are installed */
405 virtual const std::string
& get_luarocks_path() const = 0;
406 /** Set the location of where lua packages are installed */
407 virtual void set_luarocks_path(const std::string
& path
) = 0;
411 * @brief User abstraction
413 * This represents a user. In general, there will be a @a User associated with an OP
414 * (the user performing the OP), and potentially several others acting as owners.
415 * Lifetime of a User is a bit tricky , since it must last as long as any Buckets
416 * associated with it. A User has associated metadata, including a set of key/value
417 * attributes, and statistics (including usage) about the User.
422 RGWObjVersionTracker objv_tracker
;
427 User(const rgw_user
& _u
) : info() { info
.user_id
= _u
; }
428 User(const RGWUserInfo
& _i
) : info(_i
) {}
429 User(User
& _o
) = default;
430 virtual ~User() = default;
432 /** Clone a copy of this user. Used when modification is necessary of the copy */
433 virtual std::unique_ptr
<User
> clone() = 0;
434 /** List the buckets owned by a user */
435 virtual int list_buckets(const DoutPrefixProvider
* dpp
,
436 const std::string
& marker
, const std::string
& end_marker
,
437 uint64_t max
, bool need_stats
, BucketList
& buckets
,
438 optional_yield y
) = 0;
439 /** Create a new bucket owned by this user. Creates in the backing store, not just the instantiation. */
440 virtual int create_bucket(const DoutPrefixProvider
* dpp
,
442 const std::string
& zonegroup_id
,
443 rgw_placement_rule
& placement_rule
,
444 std::string
& swift_ver_location
,
445 const RGWQuotaInfo
* pquota_info
,
446 const RGWAccessControlPolicy
& policy
,
449 obj_version
& ep_objv
,
451 bool obj_lock_enabled
,
454 std::unique_ptr
<Bucket
>* bucket
,
455 optional_yield y
) = 0;
457 /** Get the display name for this User */
458 virtual std::string
& get_display_name() { return info
.display_name
; }
459 /** Get the tenant name for this User */
460 const std::string
& get_tenant() { return info
.user_id
.tenant
; }
461 /** Set the tenant name for this User */
462 void set_tenant(std::string
& _t
) { info
.user_id
.tenant
= _t
; }
463 /** Get the namespace for this User */
464 const std::string
& get_ns() { return info
.user_id
.ns
; }
465 /** Set the namespace for this User */
466 void set_ns(std::string
& _ns
) { info
.user_id
.ns
= _ns
; }
467 /** Clear the namespace for this User */
468 void clear_ns() { info
.user_id
.ns
.clear(); }
469 /** Get the full ID for this User */
470 const rgw_user
& get_id() const { return info
.user_id
; }
471 /** Get the type of this User */
472 uint32_t get_type() const { return info
.type
; }
473 /** Get the maximum number of buckets allowed for this User */
474 int32_t get_max_buckets() const { return info
.max_buckets
; }
475 /** Get the capabilities for this User */
476 const RGWUserCaps
& get_caps() const { return info
.caps
; }
477 /** Get the version tracker for this User */
478 virtual RGWObjVersionTracker
& get_version_tracker() { return objv_tracker
; }
479 /** Get the cached attributes for this User */
480 virtual Attrs
& get_attrs() { return attrs
; }
481 /** Set the cached attributes fro this User */
482 virtual void set_attrs(Attrs
& _attrs
) { attrs
= _attrs
; }
483 /** Check if a User pointer is empty */
484 static bool empty(User
* u
) { return (!u
|| u
->info
.user_id
.id
.empty()); }
485 /** Check if a User unique_pointer is empty */
486 static bool empty(std::unique_ptr
<User
>& u
) { return (!u
|| u
->info
.user_id
.id
.empty()); }
487 /** Read the User attributes from the backing Store */
488 virtual int read_attrs(const DoutPrefixProvider
* dpp
, optional_yield y
) = 0;
489 /** Set the attributes in attrs, leaving any other existing attrs set, and
490 * write them to the backing store; a merge operation */
491 virtual int merge_and_store_attrs(const DoutPrefixProvider
* dpp
, Attrs
& new_attrs
, optional_yield y
) = 0;
492 virtual int read_stats(const DoutPrefixProvider
*dpp
,
493 optional_yield y
, RGWStorageStats
* stats
,
494 ceph::real_time
* last_stats_sync
= nullptr,
495 ceph::real_time
* last_stats_update
= nullptr) = 0;
496 /** Read the User stats from the backing Store, asynchronous */
497 virtual int read_stats_async(const DoutPrefixProvider
*dpp
, RGWGetUserStats_CB
* cb
) = 0;
498 /** Flush accumulated stat changes for this User to the backing store */
499 virtual int complete_flush_stats(const DoutPrefixProvider
*dpp
, optional_yield y
) = 0;
500 /** Read detailed usage stats for this User from the backing store */
501 virtual int read_usage(const DoutPrefixProvider
*dpp
, uint64_t start_epoch
,
502 uint64_t end_epoch
, uint32_t max_entries
,
503 bool* is_truncated
, RGWUsageIter
& usage_iter
,
504 std::map
<rgw_user_bucket
, rgw_usage_log_entry
>& usage
) = 0;
505 /** Trim User usage stats to the given epoch range */
506 virtual int trim_usage(const DoutPrefixProvider
*dpp
, uint64_t start_epoch
, uint64_t end_epoch
) = 0;
508 /** Load this User from the backing store. requires ID to be set, fills all other fields. */
509 virtual int load_user(const DoutPrefixProvider
* dpp
, optional_yield y
) = 0;
510 /** Store this User to the backing store */
511 virtual int store_user(const DoutPrefixProvider
* dpp
, optional_yield y
, bool exclusive
, RGWUserInfo
* old_info
= nullptr) = 0;
512 /** Remove this User from the backing store */
513 virtual int remove_user(const DoutPrefixProvider
* dpp
, optional_yield y
) = 0;
515 /* dang temporary; will be removed when User is complete */
516 RGWUserInfo
& get_info() { return info
; }
518 friend inline std::ostream
& operator<<(std::ostream
& out
, const User
& u
) {
519 out
<< u
.info
.user_id
;
523 friend inline std::ostream
& operator<<(std::ostream
& out
, const User
* u
) {
527 out
<< u
->info
.user_id
;
531 friend inline std::ostream
& operator<<(std::ostream
& out
, const std::unique_ptr
<User
>& p
) {
540 * @brief Bucket abstraction
542 * This represents a bucket. A bucket is a container for objects. It is owned by a user, and has
543 * it's own set of metadata, including a set of key/value attributes. A bucket may not contain
544 * other buckets, only objects. Buckets have Access Control Lists (ACLs) that control what users
545 * can access the contents of the bucket, and in what ways.
551 User
* owner
= nullptr;
553 obj_version bucket_version
;
554 ceph::real_time mtime
;
559 * @brief Parameters for a bucket list operation
565 rgw_obj_key end_marker
;
567 bool enforce_ns
{true};
568 RGWAccessListFilter
* access_list_filter
{nullptr};
569 RGWBucketListNameFilter force_check_filter
;
570 bool list_versions
{false};
571 bool allow_unordered
{false};
572 int shard_id
{RGW_NO_SHARD
};
574 friend std::ostream
& operator<<(std::ostream
& out
, const ListParams
& p
) {
575 out
<< "rgw::sal::Bucket::ListParams{ prefix=\"" << p
.prefix
<<
576 "\", delim=\"" << p
.delim
<<
577 "\", marker=\"" << p
.marker
<<
578 "\", end_marker=\"" << p
.end_marker
<<
579 "\", ns=\"" << p
.ns
<<
580 "\", enforce_ns=" << p
.enforce_ns
<<
581 ", list_versions=" << p
.list_versions
<<
582 ", allow_unordered=" << p
.allow_unordered
<<
583 ", shard_id=" << p
.shard_id
<<
589 * @brief Results from a bucket list operation
592 std::vector
<rgw_bucket_dir_entry
> objs
;
593 std::map
<std::string
, bool> common_prefixes
;
594 bool is_truncated
{false};
595 rgw_obj_key next_marker
;
601 Bucket(const rgw_bucket
& _b
) { ent
.bucket
= _b
; info
.bucket
= _b
; }
602 Bucket(const RGWBucketEnt
& _e
) : ent(_e
) {
603 info
.bucket
= ent
.bucket
;
604 info
.placement_rule
= ent
.placement_rule
;
605 info
.creation_time
= ent
.creation_time
;
607 Bucket(const RGWBucketInfo
& _i
) : info(_i
) {
608 ent
.bucket
= info
.bucket
;
609 ent
.placement_rule
= info
.placement_rule
;
610 ent
.creation_time
= info
.creation_time
;
612 Bucket(const rgw_bucket
& _b
, User
* _u
) :
613 owner(_u
) { ent
.bucket
= _b
; info
.bucket
= _b
; }
614 Bucket(const RGWBucketEnt
& _e
, User
* _u
) : ent(_e
), owner(_u
) {
615 info
.bucket
= ent
.bucket
;
616 info
.placement_rule
= ent
.placement_rule
;
617 info
.creation_time
= ent
.creation_time
;
619 Bucket(const RGWBucketInfo
& _i
, User
* _u
) : info(_i
), owner(_u
) {
620 ent
.bucket
= info
.bucket
;
621 ent
.placement_rule
= info
.placement_rule
;
622 ent
.creation_time
= info
.creation_time
;
624 virtual ~Bucket() = default;
626 /** Get an @a Object belonging to this bucket */
627 virtual std::unique_ptr
<Object
> get_object(const rgw_obj_key
& key
) = 0;
628 /** List the contents of this bucket */
629 virtual int list(const DoutPrefixProvider
* dpp
, ListParams
&, int, ListResults
&, optional_yield y
) = 0;
630 /** Get the cached attributes associated with this bucket */
631 virtual Attrs
& get_attrs(void) { return attrs
; }
632 /** Set the cached attributes on this bucket */
633 virtual int set_attrs(Attrs a
) { attrs
= a
; return 0; }
634 /** Remove this bucket from the backing store */
635 virtual int remove_bucket(const DoutPrefixProvider
* dpp
, bool delete_children
, bool forward_to_master
, req_info
* req_info
, optional_yield y
) = 0;
636 /** Remove this bucket, bypassing garbage collection. May be removed */
637 virtual int remove_bucket_bypass_gc(int concurrent_max
, bool
638 keep_index_consistent
,
639 optional_yield y
, const
640 DoutPrefixProvider
*dpp
) = 0;
641 /** Get then ACL for this bucket */
642 virtual RGWAccessControlPolicy
& get_acl(void) = 0;
643 /** Set the ACL for this bucket */
644 virtual int set_acl(const DoutPrefixProvider
* dpp
, RGWAccessControlPolicy
& acl
, optional_yield y
) = 0;
647 void set_owner(rgw::sal::User
* _owner
) {
651 /** Load this bucket from the backing store. Requires the key to be set, fills other fields.
652 * If @a get_stats is true, then statistics on the bucket are also looked up. */
653 virtual int load_bucket(const DoutPrefixProvider
* dpp
, optional_yield y
, bool get_stats
= false) = 0;
654 /** Read the bucket stats from the backing Store, synchronous */
655 virtual int read_stats(const DoutPrefixProvider
*dpp
, int shard_id
,
656 std::string
* bucket_ver
, std::string
* master_ver
,
657 std::map
<RGWObjCategory
, RGWStorageStats
>& stats
,
658 std::string
* max_marker
= nullptr,
659 bool* syncstopped
= nullptr) = 0;
660 /** Read the bucket stats from the backing Store, asynchronous */
661 virtual int read_stats_async(const DoutPrefixProvider
*dpp
, int shard_id
, RGWGetBucketStats_CB
* ctx
) = 0;
662 /** Sync this bucket's stats to the owning user's stats in the backing store */
663 virtual int sync_user_stats(const DoutPrefixProvider
*dpp
, optional_yield y
) = 0;
664 /** Refresh the metadata stats (size, count, and so on) from the backing store */
665 virtual int update_container_stats(const DoutPrefixProvider
* dpp
) = 0;
666 /** Check if this bucket needs resharding, and schedule it if it does */
667 virtual int check_bucket_shards(const DoutPrefixProvider
* dpp
) = 0;
668 /** Change the owner of this bucket in the backing store */
669 virtual int chown(const DoutPrefixProvider
* dpp
, User
* new_user
, User
* old_user
, optional_yield y
, const std::string
* marker
= nullptr) = 0;
670 /** Store the cached bucket info into the backing store */
671 virtual int put_info(const DoutPrefixProvider
* dpp
, bool exclusive
, ceph::real_time mtime
) = 0;
672 /** Check to see if the given user is the owner of this bucket */
673 virtual bool is_owner(User
* user
) = 0;
674 /** Get the owner of this bucket */
675 virtual User
* get_owner(void) { return owner
; };
676 /** Get the owner of this bucket in the form of an ACLOwner object */
677 virtual ACLOwner
get_acl_owner(void) { return ACLOwner(info
.owner
); };
678 /** Check in the backing store if this bucket is empty */
679 virtual int check_empty(const DoutPrefixProvider
* dpp
, optional_yield y
) = 0;
680 /** Chec k if the given size fits within the quota */
681 virtual int check_quota(const DoutPrefixProvider
*dpp
, RGWQuotaInfo
& user_quota
, RGWQuotaInfo
& bucket_quota
, uint64_t obj_size
, optional_yield y
, bool check_size_only
= false) = 0;
682 /** Set the attributes in attrs, leaving any other existing attrs set, and
683 * write them to the backing store; a merge operation */
684 virtual int merge_and_store_attrs(const DoutPrefixProvider
* dpp
, Attrs
& new_attrs
, optional_yield y
) = 0;
685 /** Try to refresh the cached bucket info from the backing store. Used in
686 * read-modify-update loop. */
687 virtual int try_refresh_info(const DoutPrefixProvider
* dpp
, ceph::real_time
* pmtime
) = 0;
688 /** Read usage information about this bucket from the backing store */
689 virtual int read_usage(const DoutPrefixProvider
*dpp
, uint64_t start_epoch
, uint64_t end_epoch
, uint32_t max_entries
,
690 bool* is_truncated
, RGWUsageIter
& usage_iter
,
691 std::map
<rgw_user_bucket
, rgw_usage_log_entry
>& usage
) = 0;
692 /** Trim the usage information to the given epoch range */
693 virtual int trim_usage(const DoutPrefixProvider
*dpp
, uint64_t start_epoch
, uint64_t end_epoch
) = 0;
694 /** Remove objects from the bucket index of this bucket. May be removed from API */
695 virtual int remove_objs_from_index(const DoutPrefixProvider
*dpp
, std::list
<rgw_obj_index_key
>& objs_to_unlink
) = 0;
696 /** Check the state of the bucket index, and get stats from it. May be removed from API */
697 virtual int check_index(const DoutPrefixProvider
*dpp
, std::map
<RGWObjCategory
, RGWStorageStats
>& existing_stats
, std::map
<RGWObjCategory
, RGWStorageStats
>& calculated_stats
) = 0;
698 /** Rebuild the bucket index. May be removed from API */
699 virtual int rebuild_index(const DoutPrefixProvider
*dpp
) = 0;
700 /** Set a timeout on the check_index() call. May be removed from API */
701 virtual int set_tag_timeout(const DoutPrefixProvider
*dpp
, uint64_t timeout
) = 0;
702 /** Remove this specific bucket instance from the backing store. May be removed from API */
703 virtual int purge_instance(const DoutPrefixProvider
* dpp
) = 0;
705 /** Check if this instantiation is empty */
706 bool empty() const { return info
.bucket
.name
.empty(); }
707 /** Get the cached name of this bucket */
708 const std::string
& get_name() const { return info
.bucket
.name
; }
709 /** Get the cached tenant of this bucket */
710 const std::string
& get_tenant() const { return info
.bucket
.tenant
; }
711 /** Get the cached marker of this bucket */
712 const std::string
& get_marker() const { return info
.bucket
.marker
; }
713 /** Get the cached ID of this bucket */
714 const std::string
& get_bucket_id() const { return info
.bucket
.bucket_id
; }
715 /** Get the cached size of this bucket */
716 size_t get_size() const { return ent
.size
; }
717 /** Get the cached rounded size of this bucket */
718 size_t get_size_rounded() const { return ent
.size_rounded
; }
719 /** Get the cached object count of this bucket */
720 uint64_t get_count() const { return ent
.count
; }
721 /** Get the cached placement rule of this bucket */
722 rgw_placement_rule
& get_placement_rule() { return info
.placement_rule
; }
723 /** Get the cached creation time of this bucket */
724 ceph::real_time
& get_creation_time() { return info
.creation_time
; }
725 /** Get the cached modification time of this bucket */
726 ceph::real_time
& get_modification_time() { return mtime
; }
727 /** Get the cached version of this bucket */
728 obj_version
& get_version() { return bucket_version
; }
729 /** Set the cached version of this bucket */
730 void set_version(obj_version
&ver
) { bucket_version
= ver
; }
731 /** Check if this bucket is versioned */
732 bool versioned() { return info
.versioned(); }
733 /** Check if this bucket has versioning enabled */
734 bool versioning_enabled() { return info
.versioning_enabled(); }
736 /** Check if a Bucket pointer is empty */
737 static bool empty(Bucket
* b
) { return (!b
|| b
->empty()); }
738 /** Clone a copy of this bucket. Used when modification is necessary of the copy */
739 virtual std::unique_ptr
<Bucket
> clone() = 0;
741 /** Create a multipart upload in this bucket */
742 virtual std::unique_ptr
<MultipartUpload
> get_multipart_upload(
743 const std::string
& oid
,
744 std::optional
<std::string
> upload_id
=std::nullopt
,
745 ACLOwner owner
={}, ceph::real_time mtime
=real_clock::now()) = 0;
746 /** List multipart uploads currently in this bucket */
747 virtual int list_multiparts(const DoutPrefixProvider
*dpp
,
748 const std::string
& prefix
,
750 const std::string
& delim
,
751 const int& max_uploads
,
752 std::vector
<std::unique_ptr
<MultipartUpload
>>& uploads
,
753 std::map
<std::string
, bool> *common_prefixes
,
754 bool *is_truncated
) = 0;
755 /** Abort multipart uploads in a bucket */
756 virtual int abort_multiparts(const DoutPrefixProvider
* dpp
,
757 CephContext
* cct
) = 0;
759 /* dang - This is temporary, until the API is completed */
760 rgw_bucket
& get_key() { return info
.bucket
; }
761 RGWBucketInfo
& get_info() { return info
; }
763 friend inline std::ostream
& operator<<(std::ostream
& out
, const Bucket
& b
) {
764 out
<< b
.info
.bucket
;
768 friend inline std::ostream
& operator<<(std::ostream
& out
, const Bucket
* b
) {
772 out
<< b
->info
.bucket
;
776 friend inline std::ostream
& operator<<(std::ostream
& out
, const std::unique_ptr
<Bucket
>& p
) {
781 bool operator==(const Bucket
& b
) const {
782 return (info
.bucket
.tenant
== b
.info
.bucket
.tenant
) &&
783 (info
.bucket
.name
== b
.info
.bucket
.name
) &&
784 (info
.bucket
.bucket_id
== b
.info
.bucket
.bucket_id
);
786 bool operator!=(const Bucket
& b
) const {
787 return (info
.bucket
.tenant
!= b
.info
.bucket
.tenant
) ||
788 (info
.bucket
.name
!= b
.info
.bucket
.name
) ||
789 (info
.bucket
.bucket_id
!= b
.info
.bucket
.bucket_id
);
792 friend class BucketList
;
794 virtual void set_ent(RGWBucketEnt
& _ent
) { ent
= _ent
; info
.bucket
= ent
.bucket
; info
.placement_rule
= ent
.placement_rule
; }
798 * @brief A list of buckets
800 * This is the result from a bucket listing operation.
803 std::map
<std::string
, std::unique_ptr
<Bucket
>> buckets
;
807 BucketList() : buckets(), truncated(false) {}
808 BucketList(BucketList
&& _bl
) :
809 buckets(std::move(_bl
.buckets
)),
810 truncated(_bl
.truncated
)
812 BucketList
& operator=(const BucketList
&) = delete;
813 BucketList
& operator=(BucketList
&& _bl
) {
814 for (auto& ent
: _bl
.buckets
) {
815 buckets
.emplace(ent
.first
, std::move(ent
.second
));
817 truncated
= _bl
.truncated
;
821 /** Get the list of buckets. The list is a map of <bucket-name, Bucket> pairs. */
822 std::map
<std::string
, std::unique_ptr
<Bucket
>>& get_buckets() { return buckets
; }
823 /** True if the list is truncated (that is, there are more buckets to list) */
824 bool is_truncated(void) const { return truncated
; }
825 /** Set the truncated state of the list */
826 void set_truncated(bool trunc
) { truncated
= trunc
; }
827 /** Add a bucket to the list. Takes ownership of the bucket */
828 void add(std::unique_ptr
<Bucket
> bucket
) {
829 buckets
.emplace(bucket
->info
.bucket
.name
, std::move(bucket
));
831 /** The number of buckets in this list */
832 size_t count() const { return buckets
.size(); }
833 /** Clear the list */
841 * @brief Object abstraction
843 * This represents an Object. An Object is the basic unit of data storage. It
844 * represents a blob of data, a set of metadata (such as size, owner, ACLs, etc.) and
845 * a set of key/value attributes. Objects may be versioned. If a versioned object
846 * is written to, a new object with the same name but a different version is created,
847 * and the old version of the object is still accessible. If an unversioned object
848 * is written to, it is replaced, and the old data is not accessible.
854 std::string index_hash_source
;
857 ceph::real_time mtime
;
858 bool delete_marker
{false};
859 bool in_extra_data
{false};
864 * @brief Read operation on an Object
866 * This represents a Read operation on an Object. Read operations are optionally
867 * asynchronous, using the iterate() API.
871 const ceph::real_time
* mod_ptr
{nullptr};
872 const ceph::real_time
* unmod_ptr
{nullptr};
873 bool high_precision_time
{false};
874 uint32_t mod_zone_id
{0};
875 uint64_t mod_pg_ver
{0};
876 const char* if_match
{nullptr};
877 const char* if_nomatch
{nullptr};
878 ceph::real_time
* lastmod
{nullptr};
879 rgw_obj
* target_obj
{nullptr}; // XXX dang remove?
882 virtual ~ReadOp() = default;
884 /** Prepare the Read op. Must be called first */
885 virtual int prepare(optional_yield y
, const DoutPrefixProvider
* dpp
) = 0;
886 /** Synchronous read. Read from @a ofs to @a end into @a bl */
887 virtual int read(int64_t ofs
, int64_t end
, bufferlist
& bl
, optional_yield y
, const DoutPrefixProvider
* dpp
) = 0;
888 /** Asynchronous read. Read from @a ofs to @a end calling @a cb on each read
890 virtual int iterate(const DoutPrefixProvider
* dpp
, int64_t ofs
, int64_t end
, RGWGetDataCB
* cb
, optional_yield y
) = 0;
891 /** Get an attribute by name */
892 virtual int get_attr(const DoutPrefixProvider
* dpp
, const char* name
, bufferlist
& dest
, optional_yield y
) = 0;
896 * @brief Delete operation on an Object
898 * This deletes an Object from the backing store.
902 ACLOwner bucket_owner
;
904 int versioning_status
{0};
905 uint64_t olh_epoch
{0};
906 std::string marker_version_id
;
907 uint32_t bilog_flags
{0};
908 std::list
<rgw_obj_index_key
>* remove_objs
{nullptr};
909 ceph::real_time expiration_time
;
910 ceph::real_time unmod_since
;
911 ceph::real_time mtime
;
912 bool high_precision_time
{false};
913 rgw_zone_set
* zones_trace
{nullptr};
915 uint64_t parts_accounted_size
{0};
919 bool delete_marker
{false};
920 std::string version_id
;
923 virtual ~DeleteOp() = default;
925 /** Delete the object */
926 virtual int delete_obj(const DoutPrefixProvider
* dpp
, optional_yield y
) = 0;
936 Object(const rgw_obj_key
& _k
)
943 Object(const rgw_obj_key
& _k
, Bucket
* _b
)
950 Object(Object
& _o
) = default;
952 virtual ~Object() = default;
954 /** Shortcut synchronous delete call for common deletes */
955 virtual int delete_object(const DoutPrefixProvider
* dpp
,
956 RGWObjectCtx
* obj_ctx
,
958 bool prevent_versioning
= false) = 0;
959 /** Asynchronous delete call */
960 virtual int delete_obj_aio(const DoutPrefixProvider
* dpp
, RGWObjState
* astate
, Completions
* aio
,
961 bool keep_index_consistent
, optional_yield y
) = 0;
962 /** Copy an this object to another object. */
963 virtual int copy_object(RGWObjectCtx
& obj_ctx
, User
* user
,
964 req_info
* info
, const rgw_zone_id
& source_zone
,
965 rgw::sal::Object
* dest_object
, rgw::sal::Bucket
* dest_bucket
,
966 rgw::sal::Bucket
* src_bucket
,
967 const rgw_placement_rule
& dest_placement
,
968 ceph::real_time
* src_mtime
, ceph::real_time
* mtime
,
969 const ceph::real_time
* mod_ptr
, const ceph::real_time
* unmod_ptr
,
970 bool high_precision_time
,
971 const char* if_match
, const char* if_nomatch
,
972 AttrsMod attrs_mod
, bool copy_if_newer
, Attrs
& attrs
,
973 RGWObjCategory category
, uint64_t olh_epoch
,
974 boost::optional
<ceph::real_time
> delete_at
,
975 std::string
* version_id
, std::string
* tag
, std::string
* etag
,
976 void (*progress_cb
)(off_t
, void *), void* progress_data
,
977 const DoutPrefixProvider
* dpp
, optional_yield y
) = 0;
978 /** Get the ACL for this object */
979 virtual RGWAccessControlPolicy
& get_acl(void) = 0;
980 /** Set the ACL for this object */
981 virtual int set_acl(const RGWAccessControlPolicy
& acl
) = 0;
982 /** Mark further operations on this object as being atomic */
983 virtual void set_atomic(RGWObjectCtx
* rctx
) const = 0;
984 /** Pre-fetch data when reading */
985 virtual void set_prefetch_data(RGWObjectCtx
* rctx
) = 0;
986 /** Mark data as compressed */
987 virtual void set_compressed(RGWObjectCtx
* rctx
) = 0;
989 /** Check to see if this object has an empty key. This means it's uninitialized */
990 bool empty() const { return key
.empty(); }
991 /** Get the name of this object */
992 const std::string
&get_name() const { return key
.name
; }
994 /** Get the object state for this object. Will be removed in the future */
995 virtual int get_obj_state(const DoutPrefixProvider
* dpp
, RGWObjectCtx
* rctx
, RGWObjState
**state
, optional_yield y
, bool follow_olh
= true) = 0;
996 /** Set attributes for this object from the backing store. Attrs can be set or
997 * deleted. @note the attribute APIs may be revisited in the future. */
998 virtual int set_obj_attrs(const DoutPrefixProvider
* dpp
, RGWObjectCtx
* rctx
, Attrs
* setattrs
, Attrs
* delattrs
, optional_yield y
, rgw_obj
* target_obj
= NULL
) = 0;
999 /** Get attributes for this object */
1000 virtual int get_obj_attrs(RGWObjectCtx
* rctx
, optional_yield y
, const DoutPrefixProvider
* dpp
, rgw_obj
* target_obj
= NULL
) = 0;
1001 /** Modify attributes for this object. */
1002 virtual int modify_obj_attrs(RGWObjectCtx
* rctx
, const char* attr_name
, bufferlist
& attr_val
, optional_yield y
, const DoutPrefixProvider
* dpp
) = 0;
1003 /** Delete attributes for this object */
1004 virtual int delete_obj_attrs(const DoutPrefixProvider
* dpp
, RGWObjectCtx
* rctx
, const char* attr_name
, optional_yield y
) = 0;
1005 /** Check to see if this object has expired */
1006 virtual bool is_expired() = 0;
1007 /** Create a randomized instance ID for this object */
1008 virtual void gen_rand_obj_instance_name() = 0;
1009 /** Get a multipart serializer for this object */
1010 virtual MPSerializer
* get_serializer(const DoutPrefixProvider
*dpp
, const std::string
& lock_name
) = 0;
1011 /** Move the data of an object to new placement storage */
1012 virtual int transition(RGWObjectCtx
& rctx
,
1014 const rgw_placement_rule
& placement_rule
,
1015 const real_time
& mtime
,
1017 const DoutPrefixProvider
* dpp
,
1018 optional_yield y
) = 0;
1019 /** Check to see if two placement rules match */
1020 virtual bool placement_rules_match(rgw_placement_rule
& r1
, rgw_placement_rule
& r2
) = 0;
1021 /** Dump store-specific object layout info in JSON */
1022 virtual int dump_obj_layout(const DoutPrefixProvider
*dpp
, optional_yield y
, Formatter
* f
, RGWObjectCtx
* obj_ctx
) = 0;
1024 /** Get the cached attributes for this object */
1025 Attrs
& get_attrs(void) { return attrs
; }
1026 /** Get the (const) cached attributes for this object */
1027 const Attrs
& get_attrs(void) const { return attrs
; }
1028 /** Set the cached attributes for this object */
1029 virtual int set_attrs(Attrs a
) { attrs
= a
; return 0; }
1030 /** Get the cached modification time for this object */
1031 ceph::real_time
get_mtime(void) const { return mtime
; }
1032 /** Get the cached size for this object */
1033 uint64_t get_obj_size(void) const { return obj_size
; }
1034 /** Get the bucket containing this object */
1035 Bucket
* get_bucket(void) const { return bucket
; }
1036 /** Set the bucket containing this object */
1037 void set_bucket(Bucket
* b
) { bucket
= b
; }
1038 /** Get the sharding hash representation of this object */
1039 std::string
get_hash_source(void) { return index_hash_source
; }
1040 /** Set the sharding hash representation of this object */
1041 void set_hash_source(std::string s
) { index_hash_source
= s
; }
1042 /** Build an Object Identifier string for this object */
1043 std::string
get_oid(void) const { return key
.get_oid(); }
1044 /** True if this object is a delete marker (newest version is deleted) */
1045 bool get_delete_marker(void) { return delete_marker
; }
1046 /** True if this object is stored in the extra data pool */
1047 bool get_in_extra_data(void) { return in_extra_data
; }
1048 /** Set the in_extra_data field */
1049 void set_in_extra_data(bool i
) { in_extra_data
= i
; }
1050 /** Helper to sanitize object size, offset, and end values */
1051 int range_to_ofs(uint64_t obj_size
, int64_t &ofs
, int64_t &end
);
1052 /** Set the cached size of this object */
1053 void set_obj_size(uint64_t s
) { obj_size
= s
; }
1054 /** Set the cached name of this object */
1055 virtual void set_name(const std::string
& n
) { key
= n
; }
1056 /** Set the cached key of this object */
1057 virtual void set_key(const rgw_obj_key
& k
) { key
= k
; }
1058 /** Get an rgw_obj representing this object */
1059 virtual rgw_obj
get_obj(void) const {
1060 rgw_obj
obj(bucket
->get_key(), key
);
1061 obj
.set_in_extra_data(in_extra_data
);
1062 obj
.index_hash_source
= index_hash_source
;
1066 /** Restore the previous swift version of this object */
1067 virtual int swift_versioning_restore(RGWObjectCtx
* obj_ctx
,
1068 bool& restored
, /* out */
1069 const DoutPrefixProvider
* dpp
) = 0;
1070 /** Copy the current version of a swift object to the configured destination bucket*/
1071 virtual int swift_versioning_copy(RGWObjectCtx
* obj_ctx
,
1072 const DoutPrefixProvider
* dpp
,
1073 optional_yield y
) = 0;
1075 /** Get a new ReadOp for this object */
1076 virtual std::unique_ptr
<ReadOp
> get_read_op(RGWObjectCtx
*) = 0;
1077 /** Get a new DeleteOp for this object */
1078 virtual std::unique_ptr
<DeleteOp
> get_delete_op(RGWObjectCtx
*) = 0;
1080 /** Get @a count OMAP values via listing, starting at @a marker for this object */
1081 virtual int omap_get_vals(const DoutPrefixProvider
*dpp
, const std::string
& marker
, uint64_t count
,
1082 std::map
<std::string
, bufferlist
>* m
,
1083 bool* pmore
, optional_yield y
) = 0;
1084 /** Get all OMAP key/value pairs for this object */
1085 virtual int omap_get_all(const DoutPrefixProvider
*dpp
, std::map
<std::string
, bufferlist
>* m
,
1086 optional_yield y
) = 0;
1087 /** Get the OMAP values matching the given set of keys */
1088 virtual int omap_get_vals_by_keys(const DoutPrefixProvider
*dpp
, const std::string
& oid
,
1089 const std::set
<std::string
>& keys
,
1091 /** Get a single OMAP value matching the given key */
1092 virtual int omap_set_val_by_key(const DoutPrefixProvider
*dpp
, const std::string
& key
, bufferlist
& val
,
1093 bool must_exist
, optional_yield y
) = 0;
1095 /** Check to see if the give object pointer is uninitialized */
1096 static bool empty(Object
* o
) { return (!o
|| o
->empty()); }
1097 /** Get a unique copy of this object */
1098 virtual std::unique_ptr
<Object
> clone() = 0;
1100 /* dang - This is temporary, until the API is completed */
1101 /** Get the key for this object */
1102 rgw_obj_key
& get_key() { return key
; }
1103 /** Set the instance for this object */
1104 void set_instance(const std::string
&i
) { key
.set_instance(i
); }
1105 /** Get the instance for this object */
1106 const std::string
&get_instance() const { return key
.instance
; }
1107 /** Check to see if this object has an instance set */
1108 bool have_instance(void) { return key
.have_instance(); }
1110 friend inline std::ostream
& operator<<(std::ostream
& out
, const Object
& o
) {
1112 out
<< o
.bucket
<< ":";
1116 friend inline std::ostream
& operator<<(std::ostream
& out
, const Object
* o
) {
1123 friend inline std::ostream
& operator<<(std::ostream
& out
, const std::unique_ptr
<Object
>& p
) {
1130 * @brief Abstraction of a single part of a multipart upload
1132 class MultipartPart
{
1137 MultipartPart() = default;
1138 virtual ~MultipartPart() = default;
1140 /** Get the part number of this part */
1141 virtual uint32_t get_num() = 0;
1142 /** Get the size of this part */
1143 virtual uint64_t get_size() = 0;
1144 /** Get the etag of this part */
1145 virtual const std::string
& get_etag() = 0;
1146 /** Get the modification time of this part */
1147 virtual ceph::real_time
& get_mtime() = 0;
1151 * @brief Abstraction of a multipart upload
1153 * This represents a multipart upload. For large objects, it's inefficient to do a
1154 * single, long-lived upload of the object. Instead, protocols such as S3 allow the
1155 * client to start a multipart upload, and then upload object in smaller parts in
1156 * parallel. A MultipartUpload consists of a target bucket, a unique identifier, and a
1157 * set of upload parts.
1159 class MultipartUpload
{
1162 std::map
<uint32_t, std::unique_ptr
<MultipartPart
>> parts
;
1163 jspan_context trace_ctx
{false, false};
1165 MultipartUpload(Bucket
* _bucket
) : bucket(_bucket
) {}
1166 virtual ~MultipartUpload() = default;
1168 /** Get the name of the object representing this upload in the backing store */
1169 virtual const std::string
& get_meta() const = 0;
1170 /** Get the name of the target object for this upload */
1171 virtual const std::string
& get_key() const = 0;
1172 /** Get the unique ID of this upload */
1173 virtual const std::string
& get_upload_id() const = 0;
1174 /** Get the owner of this upload */
1175 virtual const ACLOwner
& get_owner() const = 0;
1176 /** Get the modification time of this upload */
1177 virtual ceph::real_time
& get_mtime() = 0;
1179 /** Get all the cached parts that make up this upload */
1180 std::map
<uint32_t, std::unique_ptr
<MultipartPart
>>& get_parts() { return parts
; }
1182 /** Get the trace context of this upload */
1183 const jspan_context
& get_trace() { return trace_ctx
; }
1185 /** Get the Object that represents this upload */
1186 virtual std::unique_ptr
<rgw::sal::Object
> get_meta_obj() = 0;
1188 /** Initialize this upload */
1189 virtual int init(const DoutPrefixProvider
* dpp
, optional_yield y
, RGWObjectCtx
* obj_ctx
, ACLOwner
& owner
, rgw_placement_rule
& dest_placement
, rgw::sal::Attrs
& attrs
) = 0;
1190 /** List all the parts of this upload, filling the parts cache */
1191 virtual int list_parts(const DoutPrefixProvider
* dpp
, CephContext
* cct
,
1192 int num_parts
, int marker
,
1193 int* next_marker
, bool* truncated
,
1194 bool assume_unsorted
= false) = 0;
1195 /** Abort this upload */
1196 virtual int abort(const DoutPrefixProvider
* dpp
, CephContext
* cct
,
1197 RGWObjectCtx
* obj_ctx
) = 0;
1198 /** Complete this upload, making it available as a normal object */
1199 virtual int complete(const DoutPrefixProvider
* dpp
,
1200 optional_yield y
, CephContext
* cct
,
1201 std::map
<int, std::string
>& part_etags
,
1202 std::list
<rgw_obj_index_key
>& remove_objs
,
1203 uint64_t& accounted_size
, bool& compressed
,
1204 RGWCompressionInfo
& cs_info
, off_t
& ofs
,
1205 std::string
& tag
, ACLOwner
& owner
,
1207 rgw::sal::Object
* target_obj
,
1208 RGWObjectCtx
* obj_ctx
) = 0;
1210 /** Get placement and/or attribute info for this upload */
1211 virtual int get_info(const DoutPrefixProvider
*dpp
, optional_yield y
, RGWObjectCtx
* obj_ctx
, rgw_placement_rule
** rule
, rgw::sal::Attrs
* attrs
= nullptr) = 0;
1213 /** Get a Writer to write to a part of this upload */
1214 virtual std::unique_ptr
<Writer
> get_writer(const DoutPrefixProvider
*dpp
,
1216 std::unique_ptr
<rgw::sal::Object
> _head_obj
,
1217 const rgw_user
& owner
, RGWObjectCtx
& obj_ctx
,
1218 const rgw_placement_rule
*ptail_placement_rule
,
1220 const std::string
& part_num_str
) = 0;
1222 friend inline std::ostream
& operator<<(std::ostream
& out
, const MultipartUpload
& u
) {
1223 out
<< u
.get_meta();
1224 if (!u
.get_upload_id().empty())
1225 out
<< ":" << u
.get_upload_id();
1228 friend inline std::ostream
& operator<<(std::ostream
& out
, const MultipartUpload
* u
) {
1235 friend inline std::ostream
& operator<<(std::ostream
& out
, const
1236 std::unique_ptr
<MultipartUpload
>& p
) {
1243 * @brief Interface of a lock/serialization
1246 Serializer() = default;
1247 virtual ~Serializer() = default;
1249 /** Try to take the lock for the given amount of time. */
1250 virtual int try_lock(const DoutPrefixProvider
*dpp
, utime_t dur
, optional_yield y
) = 0;
1251 /** Unlock the lock */
1252 virtual int unlock() = 0;
1255 /** @brief Abstraction of a serializer for multipart uploads
1257 struct MPSerializer
: Serializer
{
1260 MPSerializer() : locked(false) {}
1261 virtual ~MPSerializer() = default;
1263 void clear_locked() {
1268 /** @brief Abstraction of a serializer for Lifecycle
1270 struct LCSerializer
: Serializer
{
1272 virtual ~LCSerializer() = default;
1276 * @brief Abstraction for lifecycle processing
1278 * Lifecycle processing loops over the objects in a bucket, applying per-bucket policy
1279 * to each object. Examples of policy can be deleting after a certain amount of time,
1280 * deleting extra versions, changing the storage class, and so on.
1284 /** Head of a lifecycle run. Used for tracking parallel lifecycle runs. */
1286 time_t start_date
{0};
1290 LCHead(time_t _date
, std::string
& _marker
) : start_date(_date
), marker(_marker
) {}
1293 /** Single entry in a lifecycle run. Multiple entries can exist processing different
1297 uint64_t start_time
{0};
1300 LCEntry() = default;
1301 LCEntry(std::string
& _bucket
, uint64_t _time
, uint32_t _status
) : bucket(_bucket
), start_time(_time
), status(_status
) {}
1304 Lifecycle() = default;
1305 virtual ~Lifecycle() = default;
1307 /** Get an entry matching the given marker */
1308 virtual int get_entry(const std::string
& oid
, const std::string
& marker
, LCEntry
& entry
) = 0;
1309 /** Get the entry following the given marker */
1310 virtual int get_next_entry(const std::string
& oid
, std::string
& marker
, LCEntry
& entry
) = 0;
1311 /** Store a modified entry in then backing store */
1312 virtual int set_entry(const std::string
& oid
, const LCEntry
& entry
) = 0;
1313 /** List all known entries */
1314 virtual int list_entries(const std::string
& oid
, const std::string
& marker
,
1315 uint32_t max_entries
, std::vector
<LCEntry
>& entries
) = 0;
1316 /** Remove an entry from the backing store */
1317 virtual int rm_entry(const std::string
& oid
, const LCEntry
& entry
) = 0;
1319 virtual int get_head(const std::string
& oid
, LCHead
& head
) = 0;
1320 /** Store a modified head to the backing store */
1321 virtual int put_head(const std::string
& oid
, const LCHead
& head
) = 0;
1323 /** Get a serializer for lifecycle */
1324 virtual LCSerializer
* get_serializer(const std::string
& lock_name
, const std::string
& oid
, const std::string
& cookie
) = 0;
1328 * @brief Abstraction for a Notification event
1330 * RGW can generate notifications for various events, such as object creation or
1333 class Notification
{
1337 rgw::notify::EventType event_type
;
1340 Notification(Object
* _obj
, Object
* _src_obj
, rgw::notify::EventType _type
)
1341 : obj(_obj
), src_obj(_src_obj
), event_type(_type
)
1344 virtual ~Notification() = default;
1346 /** Indicate the start of the event associated with this notification */
1347 virtual int publish_reserve(const DoutPrefixProvider
*dpp
, RGWObjTags
* obj_tags
= nullptr) = 0;
1348 /** Indicate the successful completion of the event associated with this notification */
1349 virtual int publish_commit(const DoutPrefixProvider
* dpp
, uint64_t size
,
1350 const ceph::real_time
& mtime
, const std::string
& etag
, const std::string
& version
) = 0;
1354 * @brief Abstraction for an asynchronous writer
1356 * Writing is done through a set of filters. This allows chaining filters to do things
1357 * like compression and encryption on async writes. This is the base abstraction for
1360 class Writer
: public ObjectProcessor
{
1362 const DoutPrefixProvider
* dpp
;
1365 Writer(const DoutPrefixProvider
*_dpp
, optional_yield y
) : dpp(_dpp
) {}
1366 virtual ~Writer() = default;
1368 /** prepare to start processing object data */
1369 virtual int prepare(optional_yield y
) = 0;
1371 /** Process a buffer. Called multiple times to write different buffers. */
1372 virtual int process(bufferlist
&& data
, uint64_t offset
) = 0;
1374 /** complete the operation and make its result visible to clients */
1375 virtual int complete(size_t accounted_size
, const std::string
& etag
,
1376 ceph::real_time
*mtime
, ceph::real_time set_mtime
,
1377 std::map
<std::string
, bufferlist
>& attrs
,
1378 ceph::real_time delete_at
,
1379 const char *if_match
, const char *if_nomatch
,
1380 const std::string
*user_data
,
1381 rgw_zone_set
*zones_trace
, bool *canceled
,
1382 optional_yield y
) = 0;
1386 * @brief Abstraction of a Zone
1388 * This abstraction allows access to information about zones. This can be the zone
1389 * containing the RGW, or another zone.
1393 virtual ~Zone() = default;
1395 /** Get info about the zonegroup containing this zone */
1396 virtual const RGWZoneGroup
& get_zonegroup() = 0;
1397 /** Get info about a zonegroup by ID */
1398 virtual int get_zonegroup(const std::string
& id
, RGWZoneGroup
& zonegroup
) = 0;
1399 /** Get the parameters of this zone */
1400 virtual const RGWZoneParams
& get_params() = 0;
1401 /** Get the ID of this zone */
1402 virtual const rgw_zone_id
& get_id() = 0;
1403 /** Get info about the realm containing this zone */
1404 virtual const RGWRealm
& get_realm() = 0;
1405 /** Get the name of this zone */
1406 virtual const std::string
& get_name() const = 0;
1407 /** True if this zone is writable */
1408 virtual bool is_writeable() = 0;
1409 /** Get the URL for the endpoint for redirecting to this zone */
1410 virtual bool get_redirect_endpoint(std::string
* endpoint
) = 0;
1411 /** Check to see if the given API is supported in this zone */
1412 virtual bool has_zonegroup_api(const std::string
& api
) const = 0;
1413 /** Get the current period ID for this zone */
1414 virtual const std::string
& get_current_period_id() = 0;
1418 * @brief Abstraction of a manager for Lua scripts
1420 * RGW can load and process Lua scripts. This will handle loading/storing scripts.
1422 class LuaScriptManager
{
1424 virtual ~LuaScriptManager() = default;
1426 /** Get a script named with the given key from the backing store */
1427 virtual int get(const DoutPrefixProvider
* dpp
, optional_yield y
, const std::string
& key
, std::string
& script
) = 0;
1428 /** Put a script named with the given key to the backing store */
1429 virtual int put(const DoutPrefixProvider
* dpp
, optional_yield y
, const std::string
& key
, const std::string
& script
) = 0;
1430 /** Delete a script named with the given key from the backing store */
1431 virtual int del(const DoutPrefixProvider
* dpp
, optional_yield y
, const std::string
& key
) = 0;
1434 /** @} namespace rgw::sal in group RGWSAL */
1435 } } // namespace rgw::sal
1438 * @brief A manager for Stores
1440 * This will manage the singleton instances of the various stores. Stores come in two
1441 * varieties: Full and Raw. A full store is suitable for use in a radosgw daemon. It
1442 * has full access to the cluster, if any. A raw store is a stripped down store, used
1443 * for admin commands.
1445 class StoreManager
{
1448 /** Get a full store by service name */
1449 static rgw::sal::Store
* get_storage(const DoutPrefixProvider
* dpp
, CephContext
* cct
, const std::string svc
, bool use_gc_thread
, bool use_lc_thread
, bool quota_threads
,
1450 bool run_sync_thread
, bool run_reshard_thread
, bool use_cache
= true, bool use_gc
= true) {
1451 rgw::sal::Store
* store
= init_storage_provider(dpp
, cct
, svc
, use_gc_thread
, use_lc_thread
,
1452 quota_threads
, run_sync_thread
, run_reshard_thread
, use_cache
, use_gc
);
1455 /** Get a stripped down store by service name */
1456 static rgw::sal::Store
* get_raw_storage(const DoutPrefixProvider
* dpp
, CephContext
* cct
, const std::string svc
) {
1457 rgw::sal::Store
* store
= init_raw_storage_provider(dpp
, cct
, svc
);
1460 /** Initialize a new full Store */
1461 static rgw::sal::Store
* init_storage_provider(const DoutPrefixProvider
* dpp
, CephContext
* cct
, const std::string svc
, bool use_gc_thread
, bool use_lc_thread
, bool quota_threads
, bool run_sync_thread
, bool run_reshard_thread
, bool use_metadata_cache
, bool use_gc
);
1462 /** Initialize a new raw Store */
1463 static rgw::sal::Store
* init_raw_storage_provider(const DoutPrefixProvider
* dpp
, CephContext
* cct
, const std::string svc
);
1464 /** Close a Store when it's no longer needed */
1465 static void close_storage(rgw::sal::Store
* store
);