1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab ft=cpp
5 * Ceph - scalable distributed file system
7 * Copyright (C) 2019 Red Hat, Inc.
9 * This is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU Lesser General Public
11 * License version 2.1, as published by the Free Software
12 * Foundation. See file COPYING.
18 #include "rgw_sal_fwd.h"
21 #include "rgw_notify_event_type.h"
22 #include "common/tracer.h"
23 #include "rgw_datalog_notify.h"
24 #include "include/random.h"
27 class RGWAccessListFilter
;
29 struct rgw_user_bucket
;
31 class RGWCoroutinesManagerRegistry
;
32 class RGWBucketSyncPolicyHandler
;
33 using RGWBucketSyncPolicyHandlerRef
= std::shared_ptr
<RGWBucketSyncPolicyHandler
>;
34 class RGWDataSyncStatusManager
;
35 class RGWSyncModuleInstance
;
36 typedef std::shared_ptr
<RGWSyncModuleInstance
> RGWSyncModuleInstanceRef
;
37 class RGWCompressionInfo
;
38 struct rgw_pubsub_topics
;
39 struct rgw_pubsub_bucket_topics
;
42 using RGWBucketListNameFilter
= std::function
<bool (const std::string
&)>;
47 namespace IAM
{ struct Policy
; }
52 virtual int handle_data(bufferlist
& bl
, off_t bl_ofs
, off_t bl_len
) = 0;
54 virtual ~RGWGetDataCB() {}
58 std::string read_iter
;
61 RGWUsageIter() : index(0) {}
65 * @struct RGWClusterStat
66 * Cluster-wide usage information
68 struct RGWClusterStat
{
73 /// total available/free
79 class RGWGetBucketStats_CB
: public RefCountedObject
{
82 std::map
<RGWObjCategory
, RGWStorageStats
>* stats
;
84 explicit RGWGetBucketStats_CB(const rgw_bucket
& _bucket
) : bucket(_bucket
), stats(NULL
) {}
85 ~RGWGetBucketStats_CB() override
{}
86 virtual void handle_response(int r
) = 0;
87 virtual void set_response(std::map
<RGWObjCategory
, RGWStorageStats
>* _stats
) {
92 class RGWGetUserStats_CB
: public RefCountedObject
{
95 RGWStorageStats stats
;
97 explicit RGWGetUserStats_CB(const rgw_user
& _user
) : user(_user
) {}
98 ~RGWGetUserStats_CB() override
{}
99 virtual void handle_response(int r
) = 0;
100 virtual void set_response(RGWStorageStats
& _stats
) {
107 bool is_atomic
{false};
108 bool has_attrs
{false};
110 uint64_t size
{0}; //< size of raw object
111 uint64_t accounted_size
{0}; //< size before compression, encryption
112 ceph::real_time mtime
;
116 std::string write_tag
;
117 bool fake_tag
{false};
118 std::string shadow_obj
;
119 bool has_data
{false};
121 bool prefetch_data
{false};
122 bool keep_tail
{false};
125 uint64_t pg_ver
{false};
126 uint32_t zone_short_id
{0};
127 bool compressed
{false};
129 /* important! don't forget to update copy constructor */
131 RGWObjVersionTracker objv_tracker
;
133 std::map
<std::string
, ceph::buffer::list
> attrset
;
136 RGWObjState(const RGWObjState
& rhs
);
139 bool get_attr(std::string name
, bufferlist
& dest
) {
140 auto iter
= attrset
.find(name
);
141 if (iter
!= attrset
.end()) {
150 * @defgroup RGWSAL RGW Store Abstraction Layer
152 * The Store Abstraction Layer is an API that separates the top layer of RGW that
153 * handles client protocols (such as S3 or Swift) from the bottom layer of RGW that
154 * interacts with a backing store. It allows the creation of multiple backing stores
155 * that can co-exist with a single RGW instance, and allows the creation of stacking
156 * layers of translators that can modify operations as they pass down the stack.
157 * Examples of translators might be a cache layer, a duplication layer that copies
158 * operations to multiple stores, or a policy layer that sends some operations to one
159 * store and some to another.
161 * The basic unit of a SAL implementation is the Store. Whether an actual backing store
162 * or a translator, there will be a Store implementation that represents it. Examples
163 * are the RadosStore that communicates via RADOS with a Ceph cluster, and the DBStore
164 * that uses a SQL db (such as SQLite3) as a backing store. There is a singleton
165 * instance of each Store.
167 * Data within RGW is owned by a User. The User is the unit of authentication and
170 * Data within RGW is stored as an Object. Each Object is a single chunk of data, owned
171 * by a single User, contained within a single Bucket. It has metadata associated with
172 * it, such as size, owner, and so on, and a set of key-value attributes that can
173 * contain anything needed by the top half.
175 * Data with RGW is organized into Buckets. Each Bucket is owned by a User, and
176 * contains Objects. There is a single, flat layer of Buckets, there is no hierarchy,
177 * and each Object is contained in a single Bucket.
179 * Instantiations of SAL classes are done as unique pointers, using std::unique_ptr.
180 * Instances of these classes are acquired via getters, and it's up to the caller to
181 * manage the lifetime.
183 * @note Anything using RGWObjContext is subject to change, as that type will not be
184 * used in the final API.
190 * @brief Base abstractions and API for SAL
193 namespace rgw
{ namespace sal
{
200 #define RGW_SAL_VERSION 1
204 class RGWOIDCProvider
;
209 ATTRSMOD_REPLACE
= 1,
213 // a simple streaming data processing abstraction
215 * @brief A simple streaming data processing abstraction
217 class DataProcessor
{
219 virtual ~DataProcessor() {}
222 * @brief Consume a bufferlist in its entirety at the given object offset.
224 * An empty bufferlist is given to request that any buffered data be flushed, though this doesn't
225 * wait for completions
227 virtual int process(bufferlist
&& data
, uint64_t offset
) = 0;
231 * @brief a data consumer that writes an object in a bucket
233 class ObjectProcessor
: public DataProcessor
{
235 /** prepare to start processing object data */
236 virtual int prepare(optional_yield y
) = 0;
238 /** complete the operation and make its result visible to clients */
239 virtual int complete(size_t accounted_size
, const std::string
& etag
,
240 ceph::real_time
*mtime
, ceph::real_time set_mtime
,
241 std::map
<std::string
, bufferlist
>& attrs
,
242 ceph::real_time delete_at
,
243 const char *if_match
, const char *if_nomatch
,
244 const std::string
*user_data
,
245 rgw_zone_set
*zones_trace
, bool *canceled
,
246 optional_yield y
) = 0;
249 /** Base class for AIO completions */
253 virtual ~Completions() = default;
254 virtual int drain() = 0;
257 /** A list of key-value attributes */
258 using Attrs
= std::map
<std::string
, ceph::buffer::list
>;
261 * @brief Base singleton representing a Store or Filter
263 * The Driver is the base abstraction of the SAL layer. It represents a base storage
264 * mechanism, or a intermediate stacking layer. There is a single instance of a given
265 * Driver per RGW, and this Driver mediates all access to it's backing.
267 * A Driver contains, loosely, @a User, @a Bucket, and @a Object entities. The @a Object
268 * contains data, and it's associated metadata. The @a Bucket contains Objects, and
269 * metadata about the bucket. Both Buckets and Objects are owned by a @a User, which is
270 * the basic unit of access control.
272 * A Driver also has metadata and some global responsibilities. For example, a driver is
273 * responsible for managing the LifeCycle activities for it's data.
278 virtual ~Driver() = default;
280 /** Post-creation initialization of driver */
281 virtual int initialize(CephContext
*cct
, const DoutPrefixProvider
*dpp
) = 0;
282 /** Name of this driver provider (e.g., "rados") */
283 virtual const std::string
get_name() const = 0;
284 /** Get cluster unique identifier */
285 virtual std::string
get_cluster_id(const DoutPrefixProvider
* dpp
, optional_yield y
) = 0;
286 /** Get a User from a rgw_user. Does not query driver for user info, so quick */
287 virtual std::unique_ptr
<User
> get_user(const rgw_user
& u
) = 0;
288 /** Lookup a User by access key. Queries driver for user info. */
289 virtual int get_user_by_access_key(const DoutPrefixProvider
* dpp
, const std::string
& key
, optional_yield y
, std::unique_ptr
<User
>* user
) = 0;
290 /** Lookup a User by email address. Queries driver for user info. */
291 virtual int get_user_by_email(const DoutPrefixProvider
* dpp
, const std::string
& email
, optional_yield y
, std::unique_ptr
<User
>* user
) = 0;
292 /** Lookup a User by swift username. Queries driver for user info. */
293 virtual int get_user_by_swift(const DoutPrefixProvider
* dpp
, const std::string
& user_str
, optional_yield y
, std::unique_ptr
<User
>* user
) = 0;
294 /** Get a basic Object. This Object is not looked up, and is incomplete, since is
295 * does not have a bucket. This should only be used when an Object is needed before
296 * there is a Bucket, otherwise use the get_object() in the Bucket class. */
297 virtual std::unique_ptr
<Object
> get_object(const rgw_obj_key
& k
) = 0;
298 /** Get a Bucket by info. Does not query the driver, just uses the give bucket info. */
299 virtual int get_bucket(User
* u
, const RGWBucketInfo
& i
, std::unique_ptr
<Bucket
>* bucket
) = 0;
300 /** Lookup a Bucket by key. Queries driver for bucket info. */
301 virtual int get_bucket(const DoutPrefixProvider
* dpp
, User
* u
, const rgw_bucket
& b
, std::unique_ptr
<Bucket
>* bucket
, optional_yield y
) = 0;
302 /** Lookup a Bucket by name. Queries driver for bucket info. */
303 virtual int get_bucket(const DoutPrefixProvider
* dpp
, User
* u
, const std::string
& tenant
, const std::string
& name
, std::unique_ptr
<Bucket
>* bucket
, optional_yield y
) = 0;
304 /** For multisite, this driver is the zone's master */
305 virtual bool is_meta_master() = 0;
306 /** For multisite, forward an OP to the zone's master */
307 virtual int forward_request_to_master(const DoutPrefixProvider
*dpp
, User
* user
, obj_version
* objv
,
308 bufferlist
& in_data
, JSONParser
* jp
, req_info
& info
,
309 optional_yield y
) = 0;
310 virtual int forward_iam_request_to_master(const DoutPrefixProvider
*dpp
, const RGWAccessKey
& key
, obj_version
* objv
,
312 RGWXMLDecoder::XMLParser
* parser
, req_info
& info
,
313 optional_yield y
) = 0;
314 /** Get zone info for this driver */
315 virtual Zone
* get_zone() = 0;
316 /** Get a unique ID specific to this zone. */
317 virtual std::string
zone_unique_id(uint64_t unique_num
) = 0;
318 /** Get a unique Swift transaction ID specific to this zone */
319 virtual std::string
zone_unique_trans_id(const uint64_t unique_num
) = 0;
320 /** Lookup a zonegroup by ID */
321 virtual int get_zonegroup(const std::string
& id
, std::unique_ptr
<ZoneGroup
>* zonegroup
) = 0;
322 /** List all zones in all zone groups by ID */
323 virtual int list_all_zones(const DoutPrefixProvider
* dpp
, std::list
<std::string
>& zone_ids
) = 0;
324 /** Get statistics about the cluster represented by this driver */
325 virtual int cluster_stat(RGWClusterStat
& stats
) = 0;
326 /** Get a @a Lifecycle object. Used to manage/run lifecycle transitions */
327 virtual std::unique_ptr
<Lifecycle
> get_lifecycle(void) = 0;
328 /** Get a @a Completions object. Used for Async I/O tracking */
329 virtual std::unique_ptr
<Completions
> get_completions(void) = 0;
331 /** Get a @a Notification object. Used to communicate with non-RGW daemons, such as
332 * management/tracking software */
334 virtual std::unique_ptr
<Notification
> get_notification(rgw::sal::Object
* obj
, rgw::sal::Object
* src_obj
, req_state
* s
,
335 rgw::notify::EventType event_type
, optional_yield y
, const std::string
* object_name
=nullptr) = 0;
336 /** No-req_state variant (e.g., rgwlc) */
337 virtual std::unique_ptr
<Notification
> get_notification(
338 const DoutPrefixProvider
* dpp
, rgw::sal::Object
* obj
, rgw::sal::Object
* src_obj
,
339 rgw::notify::EventType event_type
, rgw::sal::Bucket
* _bucket
, std::string
& _user_id
, std::string
& _user_tenant
,
340 std::string
& _req_id
, optional_yield y
) = 0;
341 /** Read the topic config entry into @a data and (optionally) @a objv_tracker */
342 virtual int read_topics(const std::string
& tenant
, rgw_pubsub_topics
& topics
, RGWObjVersionTracker
* objv_tracker
,
343 optional_yield y
, const DoutPrefixProvider
*dpp
) = 0;
344 /** Write @a info and (optionally) @a objv_tracker into the config */
345 virtual int write_topics(const std::string
& tenant
, const rgw_pubsub_topics
& topics
, RGWObjVersionTracker
* objv_tracker
,
346 optional_yield y
, const DoutPrefixProvider
*dpp
) = 0;
347 /** Remove the topic config, optionally a specific version */
348 virtual int remove_topics(const std::string
& tenant
, RGWObjVersionTracker
* objv_tracker
,
349 optional_yield y
,const DoutPrefixProvider
*dpp
) = 0;
350 /** Get access to the lifecycle management thread */
351 virtual RGWLC
* get_rgwlc(void) = 0;
352 /** Get access to the coroutine registry. Used to create new coroutine managers */
353 virtual RGWCoroutinesManagerRegistry
* get_cr_registry() = 0;
355 /** Log usage data to the driver. Usage data is things like bytes sent/received and
357 virtual int log_usage(const DoutPrefixProvider
*dpp
, std::map
<rgw_user_bucket
, RGWUsageBatch
>& usage_info
) = 0;
358 /** Log OP data to the driver. Data is opaque to SAL */
359 virtual int log_op(const DoutPrefixProvider
*dpp
, std::string
& oid
, bufferlist
& bl
) = 0;
360 /** Register this driver to the service map. Somewhat Rados specific; may be removed*/
361 virtual int register_to_service_map(const DoutPrefixProvider
*dpp
, const std::string
& daemon_type
,
362 const std::map
<std::string
, std::string
>& meta
) = 0;
363 /** Get default quota info. Used as fallback if a user or bucket has no quota set*/
364 virtual void get_quota(RGWQuota
& quota
) = 0;
365 /** Get global rate limit configuration*/
366 virtual void get_ratelimit(RGWRateLimitInfo
& bucket_ratelimit
, RGWRateLimitInfo
& user_ratelimit
, RGWRateLimitInfo
& anon_ratelimit
) = 0;
367 /** Enable or disable a set of bucket. e.g. if a User is suspended */
368 virtual int set_buckets_enabled(const DoutPrefixProvider
* dpp
, std::vector
<rgw_bucket
>& buckets
, bool enabled
) = 0;
369 /** Get a new request ID */
370 virtual uint64_t get_new_req_id() = 0;
371 /** Get a handler for bucket sync policy. */
372 virtual int get_sync_policy_handler(const DoutPrefixProvider
* dpp
,
373 std::optional
<rgw_zone_id
> zone
,
374 std::optional
<rgw_bucket
> bucket
,
375 RGWBucketSyncPolicyHandlerRef
* phandler
,
376 optional_yield y
) = 0;
377 /** Get a status manager for bucket sync */
378 virtual RGWDataSyncStatusManager
* get_data_sync_manager(const rgw_zone_id
& source_zone
) = 0;
379 /** Wake up sync threads for bucket metadata sync */
380 virtual void wakeup_meta_sync_shards(std::set
<int>& shard_ids
) = 0;
381 /** Wake up sync threads for bucket data sync */
382 virtual void wakeup_data_sync_shards(const DoutPrefixProvider
*dpp
, const rgw_zone_id
& source_zone
, boost::container::flat_map
<int, boost::container::flat_set
<rgw_data_notify_entry
>>& shard_ids
) = 0;
383 /** Clear all usage statistics globally */
384 virtual int clear_usage(const DoutPrefixProvider
*dpp
) = 0;
385 /** Get usage statistics for all users and buckets */
386 virtual int read_all_usage(const DoutPrefixProvider
*dpp
, uint64_t start_epoch
, uint64_t end_epoch
,
387 uint32_t max_entries
, bool* is_truncated
,
388 RGWUsageIter
& usage_iter
,
389 std::map
<rgw_user_bucket
, rgw_usage_log_entry
>& usage
) = 0;
390 /** Trim usage log for all users and buckets */
391 virtual int trim_all_usage(const DoutPrefixProvider
*dpp
, uint64_t start_epoch
, uint64_t end_epoch
) = 0;
392 /** Get a configuration value for the given name */
393 virtual int get_config_key_val(std::string name
, bufferlist
* bl
) = 0;
394 /** Start a metadata listing of the given section */
395 virtual int meta_list_keys_init(const DoutPrefixProvider
*dpp
, const std::string
& section
, const std::string
& marker
, void** phandle
) = 0;
396 /** Get the next key from a metadata list */
397 virtual int meta_list_keys_next(const DoutPrefixProvider
*dpp
, void* handle
, int max
, std::list
<std::string
>& keys
, bool* truncated
) = 0;
398 /** Complete a metadata listing */
399 virtual void meta_list_keys_complete(void* handle
) = 0;
400 /** Get the marker associated with the current metadata listing */
401 virtual std::string
meta_get_marker(void* handle
) = 0;
402 /** Remove a specific metadata key */
403 virtual int meta_remove(const DoutPrefixProvider
* dpp
, std::string
& metadata_key
, optional_yield y
) = 0;
404 /** Get an instance of the Sync module for bucket sync */
405 virtual const RGWSyncModuleInstanceRef
& get_sync_module() = 0;
406 /** Get the ID of the current host */
407 virtual std::string
get_host_id() = 0;
408 /** Get a Lua script manager for running lua scripts */
409 virtual std::unique_ptr
<LuaManager
> get_lua_manager() = 0;
410 /** Get an IAM Role by name etc. */
411 virtual std::unique_ptr
<RGWRole
> get_role(std::string name
,
414 std::string trust_policy
="",
415 std::string max_session_duration_str
="",
416 std::multimap
<std::string
,std::string
> tags
={}) = 0;
417 /** Get an IAM Role by ID */
418 virtual std::unique_ptr
<RGWRole
> get_role(std::string id
) = 0;
419 virtual std::unique_ptr
<RGWRole
> get_role(const RGWRoleInfo
& info
) = 0;
420 /** Get all IAM Roles optionally filtered by path */
421 virtual int get_roles(const DoutPrefixProvider
*dpp
,
423 const std::string
& path_prefix
,
424 const std::string
& tenant
,
425 std::vector
<std::unique_ptr
<RGWRole
>>& roles
) = 0;
426 /** Get an empty Open ID Connector provider */
427 virtual std::unique_ptr
<RGWOIDCProvider
> get_oidc_provider() = 0;
428 /** Get all Open ID Connector providers, optionally filtered by tenant */
429 virtual int get_oidc_providers(const DoutPrefixProvider
*dpp
,
430 const std::string
& tenant
,
431 std::vector
<std::unique_ptr
<RGWOIDCProvider
>>& providers
) = 0;
432 /** Get a Writer that appends to an object */
433 virtual std::unique_ptr
<Writer
> get_append_writer(const DoutPrefixProvider
*dpp
,
435 rgw::sal::Object
* obj
,
436 const rgw_user
& owner
,
437 const rgw_placement_rule
*ptail_placement_rule
,
438 const std::string
& unique_tag
,
440 uint64_t *cur_accounted_size
) = 0;
441 /** Get a Writer that atomically writes an entire object */
442 virtual std::unique_ptr
<Writer
> get_atomic_writer(const DoutPrefixProvider
*dpp
,
444 rgw::sal::Object
* obj
,
445 const rgw_user
& owner
,
446 const rgw_placement_rule
*ptail_placement_rule
,
448 const std::string
& unique_tag
) = 0;
450 /** Get the compression type of a placement rule */
451 virtual const std::string
& get_compression_type(const rgw_placement_rule
& rule
) = 0;
452 /** Check to see if this placement rule is valid */
453 virtual bool valid_placement(const rgw_placement_rule
& rule
) = 0;
455 /** Clean up a driver for termination */
456 virtual void finalize(void) = 0;
458 /** Get the Ceph context associated with this driver. May be removed. */
459 virtual CephContext
* ctx(void) = 0;
461 /** Register admin APIs unique to this driver */
462 virtual void register_admin_apis(RGWRESTMgr
* mgr
) = 0;
466 * @brief User abstraction
468 * This represents a user. In general, there will be a @a User associated with an OP
469 * (the user performing the OP), and potentially several others acting as owners.
470 * Lifetime of a User is a bit tricky , since it must last as long as any Buckets
471 * associated with it. A User has associated metadata, including a set of key/value
472 * attributes, and statistics (including usage) about the User.
477 virtual ~User() = default;
479 /** Clone a copy of this user. Used when modification is necessary of the copy */
480 virtual std::unique_ptr
<User
> clone() = 0;
481 /** List the buckets owned by a user */
482 virtual int list_buckets(const DoutPrefixProvider
* dpp
,
483 const std::string
& marker
, const std::string
& end_marker
,
484 uint64_t max
, bool need_stats
, BucketList
& buckets
,
485 optional_yield y
) = 0;
486 /** Create a new bucket owned by this user. Creates in the backing store, not just the instantiation. */
487 virtual int create_bucket(const DoutPrefixProvider
* dpp
,
489 const std::string
& zonegroup_id
,
490 rgw_placement_rule
& placement_rule
,
491 std::string
& swift_ver_location
,
492 const RGWQuotaInfo
* pquota_info
,
493 const RGWAccessControlPolicy
& policy
,
496 obj_version
& ep_objv
,
498 bool obj_lock_enabled
,
501 std::unique_ptr
<Bucket
>* bucket
,
502 optional_yield y
) = 0;
504 /** Get the display name for this User */
505 virtual std::string
& get_display_name() = 0;
506 /** Get the tenant name for this User */
507 virtual const std::string
& get_tenant() = 0;
508 /** Set the tenant name for this User */
509 virtual void set_tenant(std::string
& _t
) = 0;
510 /** Get the namespace for this User */
511 virtual const std::string
& get_ns() = 0;
512 /** Set the namespace for this User */
513 virtual void set_ns(std::string
& _ns
) = 0;
514 /** Clear the namespace for this User */
515 virtual void clear_ns() = 0;
516 /** Get the full ID for this User */
517 virtual const rgw_user
& get_id() const = 0;
518 /** Get the type of this User */
519 virtual uint32_t get_type() const = 0;
520 /** Get the maximum number of buckets allowed for this User */
521 virtual int32_t get_max_buckets() const = 0;
522 /** Get the capabilities for this User */
523 virtual const RGWUserCaps
& get_caps() const = 0;
524 /** Get the version tracker for this User */
525 virtual RGWObjVersionTracker
& get_version_tracker() = 0;
526 /** Get the cached attributes for this User */
527 virtual Attrs
& get_attrs() = 0;
528 /** Set the cached attributes fro this User */
529 virtual void set_attrs(Attrs
& _attrs
) = 0;
530 /** Check if a User is empty */
531 virtual bool empty() const = 0;
532 /** Check if a User pointer is empty */
533 static bool empty(const User
* u
) { return (!u
|| u
->empty()); }
534 /** Check if a User unique_pointer is empty */
535 static bool empty(const std::unique_ptr
<User
>& u
) { return (!u
|| u
->empty()); }
536 /** Read the User attributes from the backing Store */
537 virtual int read_attrs(const DoutPrefixProvider
* dpp
, optional_yield y
) = 0;
538 /** Set the attributes in attrs, leaving any other existing attrs set, and
539 * write them to the backing store; a merge operation */
540 virtual int merge_and_store_attrs(const DoutPrefixProvider
* dpp
, Attrs
& new_attrs
, optional_yield y
) = 0;
541 /** Read the User stats from the backing Store, synchronous */
542 virtual int read_stats(const DoutPrefixProvider
*dpp
,
543 optional_yield y
, RGWStorageStats
* stats
,
544 ceph::real_time
* last_stats_sync
= nullptr,
545 ceph::real_time
* last_stats_update
= nullptr) = 0;
546 /** Read the User stats from the backing Store, asynchronous */
547 virtual int read_stats_async(const DoutPrefixProvider
*dpp
, RGWGetUserStats_CB
* cb
) = 0;
548 /** Flush accumulated stat changes for this User to the backing store */
549 virtual int complete_flush_stats(const DoutPrefixProvider
*dpp
, optional_yield y
) = 0;
550 /** Read detailed usage stats for this User from the backing store */
551 virtual int read_usage(const DoutPrefixProvider
*dpp
, uint64_t start_epoch
,
552 uint64_t end_epoch
, uint32_t max_entries
,
553 bool* is_truncated
, RGWUsageIter
& usage_iter
,
554 std::map
<rgw_user_bucket
, rgw_usage_log_entry
>& usage
) = 0;
555 /** Trim User usage stats to the given epoch range */
556 virtual int trim_usage(const DoutPrefixProvider
*dpp
, uint64_t start_epoch
, uint64_t end_epoch
) = 0;
558 /** Load this User from the backing store. requires ID to be set, fills all other fields. */
559 virtual int load_user(const DoutPrefixProvider
* dpp
, optional_yield y
) = 0;
560 /** Store this User to the backing store */
561 virtual int store_user(const DoutPrefixProvider
* dpp
, optional_yield y
, bool exclusive
, RGWUserInfo
* old_info
= nullptr) = 0;
562 /** Remove this User from the backing store */
563 virtual int remove_user(const DoutPrefixProvider
* dpp
, optional_yield y
) = 0;
564 /** Verify multi-factor authentication for this user */
565 virtual int verify_mfa(const std::string
& mfa_str
, bool* verified
, const DoutPrefixProvider
* dpp
, optional_yield y
) = 0;
567 /* dang temporary; will be removed when User is complete */
568 virtual RGWUserInfo
& get_info() = 0;
570 /** Print the User to @a out */
571 virtual void print(std::ostream
& out
) const = 0;
573 friend inline std::ostream
& operator<<(std::ostream
& out
, const User
& u
) {
578 friend inline std::ostream
& operator<<(std::ostream
& out
, const User
* u
) {
586 friend inline std::ostream
& operator<<(std::ostream
& out
, const std::unique_ptr
<User
>& p
) {
593 * @brief Bucket abstraction
595 * This represents a bucket. A bucket is a container for objects. It is owned by a user, and has
596 * it's own set of metadata, including a set of key/value attributes. A bucket may not contain
597 * other buckets, only objects. Buckets have Access Control Lists (ACLs) that control what users
598 * can access the contents of the bucket, and in what ways.
604 * @brief Parameters for a bucket list operation
610 rgw_obj_key end_marker
;
612 bool enforce_ns
{true};
613 RGWAccessListFilter
* access_list_filter
{nullptr};
614 RGWBucketListNameFilter force_check_filter
;
615 bool list_versions
{false};
616 bool allow_unordered
{false};
617 int shard_id
{RGW_NO_SHARD
};
619 friend std::ostream
& operator<<(std::ostream
& out
, const ListParams
& p
) {
620 out
<< "rgw::sal::Bucket::ListParams{ prefix=\"" << p
.prefix
<<
621 "\", delim=\"" << p
.delim
<<
622 "\", marker=\"" << p
.marker
<<
623 "\", end_marker=\"" << p
.end_marker
<<
624 "\", ns=\"" << p
.ns
<<
625 "\", enforce_ns=" << p
.enforce_ns
<<
626 ", list_versions=" << p
.list_versions
<<
627 ", allow_unordered=" << p
.allow_unordered
<<
628 ", shard_id=" << p
.shard_id
<<
634 * @brief Results from a bucket list operation
637 std::vector
<rgw_bucket_dir_entry
> objs
;
638 std::map
<std::string
, bool> common_prefixes
;
639 bool is_truncated
{false};
640 rgw_obj_key next_marker
;
644 virtual ~Bucket() = default;
646 /** Get an @a Object belonging to this bucket */
647 virtual std::unique_ptr
<Object
> get_object(const rgw_obj_key
& key
) = 0;
648 /** List the contents of this bucket */
649 virtual int list(const DoutPrefixProvider
* dpp
, ListParams
&, int, ListResults
&, optional_yield y
) = 0;
650 /** Get the cached attributes associated with this bucket */
651 virtual Attrs
& get_attrs(void) = 0;
652 /** Set the cached attributes on this bucket */
653 virtual int set_attrs(Attrs a
) = 0;
654 /** Remove this bucket from the backing store */
655 virtual int remove_bucket(const DoutPrefixProvider
* dpp
, bool delete_children
, bool forward_to_master
, req_info
* req_info
, optional_yield y
) = 0;
656 /** Remove this bucket, bypassing garbage collection. May be removed */
657 virtual int remove_bucket_bypass_gc(int concurrent_max
, bool
658 keep_index_consistent
,
659 optional_yield y
, const
660 DoutPrefixProvider
*dpp
) = 0;
661 /** Get then ACL for this bucket */
662 virtual RGWAccessControlPolicy
& get_acl(void) = 0;
663 /** Set the ACL for this bucket */
664 virtual int set_acl(const DoutPrefixProvider
* dpp
, RGWAccessControlPolicy
& acl
, optional_yield y
) = 0;
667 virtual void set_owner(rgw::sal::User
* _owner
) = 0;
669 /** Load this bucket from the backing store. Requires the key to be set, fills other fields.
670 * If @a get_stats is true, then statistics on the bucket are also looked up. */
671 virtual int load_bucket(const DoutPrefixProvider
* dpp
, optional_yield y
, bool get_stats
= false) = 0;
672 /** Read the bucket stats from the backing Store, synchronous */
673 virtual int read_stats(const DoutPrefixProvider
*dpp
,
674 const bucket_index_layout_generation
& idx_layout
,
675 int shard_id
, std::string
* bucket_ver
, std::string
* master_ver
,
676 std::map
<RGWObjCategory
, RGWStorageStats
>& stats
,
677 std::string
* max_marker
= nullptr,
678 bool* syncstopped
= nullptr) = 0;
679 /** Read the bucket stats from the backing Store, asynchronous */
680 virtual int read_stats_async(const DoutPrefixProvider
*dpp
,
681 const bucket_index_layout_generation
& idx_layout
,
682 int shard_id
, RGWGetBucketStats_CB
* ctx
) = 0;
683 /** Sync this bucket's stats to the owning user's stats in the backing store */
684 virtual int sync_user_stats(const DoutPrefixProvider
*dpp
, optional_yield y
) = 0;
685 /** Refresh the metadata stats (size, count, and so on) from the backing store */
686 virtual int update_container_stats(const DoutPrefixProvider
* dpp
) = 0;
687 /** Check if this bucket needs resharding, and schedule it if it does */
688 virtual int check_bucket_shards(const DoutPrefixProvider
* dpp
) = 0;
689 /** Change the owner of this bucket in the backing store. Current owner must be set. Does not
690 * change ownership of the objects in the bucket. */
691 virtual int chown(const DoutPrefixProvider
* dpp
, User
& new_user
, optional_yield y
) = 0;
692 /** Store the cached bucket info into the backing store */
693 virtual int put_info(const DoutPrefixProvider
* dpp
, bool exclusive
, ceph::real_time mtime
) = 0;
694 /** Check to see if the given user is the owner of this bucket */
695 virtual bool is_owner(User
* user
) = 0;
696 /** Get the owner of this bucket */
697 virtual User
* get_owner(void) = 0;
698 /** Get the owner of this bucket in the form of an ACLOwner object */
699 virtual ACLOwner
get_acl_owner(void) = 0;
700 /** Check in the backing store if this bucket is empty */
701 virtual int check_empty(const DoutPrefixProvider
* dpp
, optional_yield y
) = 0;
702 /** Chec k if the given size fits within the quota */
703 virtual int check_quota(const DoutPrefixProvider
*dpp
, RGWQuota
& quota
, uint64_t obj_size
, optional_yield y
, bool check_size_only
= false) = 0;
704 /** Set the attributes in attrs, leaving any other existing attrs set, and
705 * write them to the backing store; a merge operation */
706 virtual int merge_and_store_attrs(const DoutPrefixProvider
* dpp
, Attrs
& new_attrs
, optional_yield y
) = 0;
707 /** Try to refresh the cached bucket info from the backing store. Used in
708 * read-modify-update loop. */
709 virtual int try_refresh_info(const DoutPrefixProvider
* dpp
, ceph::real_time
* pmtime
) = 0;
710 /** Read usage information about this bucket from the backing store */
711 virtual int read_usage(const DoutPrefixProvider
*dpp
, uint64_t start_epoch
, uint64_t end_epoch
, uint32_t max_entries
,
712 bool* is_truncated
, RGWUsageIter
& usage_iter
,
713 std::map
<rgw_user_bucket
, rgw_usage_log_entry
>& usage
) = 0;
714 /** Trim the usage information to the given epoch range */
715 virtual int trim_usage(const DoutPrefixProvider
*dpp
, uint64_t start_epoch
, uint64_t end_epoch
) = 0;
716 /** Remove objects from the bucket index of this bucket. May be removed from API */
717 virtual int remove_objs_from_index(const DoutPrefixProvider
*dpp
, std::list
<rgw_obj_index_key
>& objs_to_unlink
) = 0;
718 /** Check the state of the bucket index, and get stats from it. May be removed from API */
719 virtual int check_index(const DoutPrefixProvider
*dpp
, std::map
<RGWObjCategory
, RGWStorageStats
>& existing_stats
, std::map
<RGWObjCategory
, RGWStorageStats
>& calculated_stats
) = 0;
720 /** Rebuild the bucket index. May be removed from API */
721 virtual int rebuild_index(const DoutPrefixProvider
*dpp
) = 0;
722 /** Set a timeout on the check_index() call. May be removed from API */
723 virtual int set_tag_timeout(const DoutPrefixProvider
*dpp
, uint64_t timeout
) = 0;
724 /** Remove this specific bucket instance from the backing store. May be removed from API */
725 virtual int purge_instance(const DoutPrefixProvider
* dpp
) = 0;
727 /** Check if this instantiation is empty */
728 virtual bool empty() const = 0;
729 /** Get the cached name of this bucket */
730 virtual const std::string
& get_name() const = 0;
731 /** Get the cached tenant of this bucket */
732 virtual const std::string
& get_tenant() const = 0;
733 /** Get the cached marker of this bucket */
734 virtual const std::string
& get_marker() const = 0;
735 /** Get the cached ID of this bucket */
736 virtual const std::string
& get_bucket_id() const = 0;
737 /** Get the cached size of this bucket */
738 virtual size_t get_size() const = 0;
739 /** Get the cached rounded size of this bucket */
740 virtual size_t get_size_rounded() const = 0;
741 /** Get the cached object count of this bucket */
742 virtual uint64_t get_count() const = 0;
743 /** Get the cached placement rule of this bucket */
744 virtual rgw_placement_rule
& get_placement_rule() = 0;
745 /** Get the cached creation time of this bucket */
746 virtual ceph::real_time
& get_creation_time() = 0;
747 /** Get the cached modification time of this bucket */
748 virtual ceph::real_time
& get_modification_time() = 0;
749 /** Get the cached version of this bucket */
750 virtual obj_version
& get_version() = 0;
751 /** Set the cached version of this bucket */
752 virtual void set_version(obj_version
&ver
) = 0;
753 /** Check if this bucket is versioned */
754 virtual bool versioned() = 0;
755 /** Check if this bucket has versioning enabled */
756 virtual bool versioning_enabled() = 0;
758 /** Check if a Bucket pointer is empty */
759 static bool empty(const Bucket
* b
) { return (!b
|| b
->empty()); }
760 /** Check if a Bucket unique pointer is empty */
761 static bool empty(const std::unique_ptr
<Bucket
>& b
) { return (!b
|| b
->empty()); }
762 /** Clone a copy of this bucket. Used when modification is necessary of the copy */
763 virtual std::unique_ptr
<Bucket
> clone() = 0;
765 /** Create a multipart upload in this bucket */
766 virtual std::unique_ptr
<MultipartUpload
> get_multipart_upload(
767 const std::string
& oid
,
768 std::optional
<std::string
> upload_id
=std::nullopt
,
769 ACLOwner owner
={}, ceph::real_time mtime
=real_clock::now()) = 0;
770 /** List multipart uploads currently in this bucket */
771 virtual int list_multiparts(const DoutPrefixProvider
*dpp
,
772 const std::string
& prefix
,
774 const std::string
& delim
,
775 const int& max_uploads
,
776 std::vector
<std::unique_ptr
<MultipartUpload
>>& uploads
,
777 std::map
<std::string
, bool> *common_prefixes
,
778 bool *is_truncated
) = 0;
779 /** Abort multipart uploads in a bucket */
780 virtual int abort_multiparts(const DoutPrefixProvider
* dpp
,
781 CephContext
* cct
) = 0;
783 /** Read the bucket notification config into @a notifications with and (optionally) @a objv_tracker */
784 virtual int read_topics(rgw_pubsub_bucket_topics
& notifications
,
785 RGWObjVersionTracker
* objv_tracker
, optional_yield y
, const DoutPrefixProvider
*dpp
) = 0;
786 /** Write @a notifications with (optionally) @a objv_tracker into the bucket notification config */
787 virtual int write_topics(const rgw_pubsub_bucket_topics
& notifications
, RGWObjVersionTracker
* objv_tracker
,
788 optional_yield y
, const DoutPrefixProvider
*dpp
) = 0;
789 /** Remove the bucket notification config with (optionally) @a objv_tracker */
790 virtual int remove_topics(RGWObjVersionTracker
* objv_tracker
,
791 optional_yield y
, const DoutPrefixProvider
*dpp
) = 0;
793 /* dang - This is temporary, until the API is completed */
794 virtual rgw_bucket
& get_key() = 0;
795 virtual RGWBucketInfo
& get_info() = 0;
797 /** Print the User to @a out */
798 virtual void print(std::ostream
& out
) const = 0;
800 friend inline std::ostream
& operator<<(std::ostream
& out
, const Bucket
& b
) {
805 friend inline std::ostream
& operator<<(std::ostream
& out
, const Bucket
* b
) {
813 friend inline std::ostream
& operator<<(std::ostream
& out
, const std::unique_ptr
<Bucket
>& p
) {
818 virtual bool operator==(const Bucket
& b
) const = 0;
819 virtual bool operator!=(const Bucket
& b
) const = 0;
821 friend class BucketList
;
825 * @brief A list of buckets
827 * This is the result from a bucket listing operation.
830 std::map
<std::string
, std::unique_ptr
<Bucket
>> buckets
;
834 BucketList() : buckets(), truncated(false) {}
835 BucketList(BucketList
&& _bl
) :
836 buckets(std::move(_bl
.buckets
)),
837 truncated(_bl
.truncated
)
839 BucketList
& operator=(const BucketList
&) = delete;
840 BucketList
& operator=(BucketList
&& _bl
) {
841 for (auto& ent
: _bl
.buckets
) {
842 buckets
.emplace(ent
.first
, std::move(ent
.second
));
844 truncated
= _bl
.truncated
;
848 /** Get the list of buckets. The list is a map of <bucket-name, Bucket> pairs. */
849 std::map
<std::string
, std::unique_ptr
<Bucket
>>& get_buckets() { return buckets
; }
850 /** True if the list is truncated (that is, there are more buckets to list) */
851 bool is_truncated(void) const { return truncated
; }
852 /** Set the truncated state of the list */
853 void set_truncated(bool trunc
) { truncated
= trunc
; }
854 /** Add a bucket to the list. Takes ownership of the bucket */
855 void add(std::unique_ptr
<Bucket
> bucket
) {
856 buckets
.emplace(bucket
->get_name(), std::move(bucket
));
858 /** The number of buckets in this list */
859 size_t count() const { return buckets
.size(); }
860 /** Clear the list */
868 * @brief Object abstraction
870 * This represents an Object. An Object is the basic unit of data storage. It
871 * represents a blob of data, a set of metadata (such as size, owner, ACLs, etc.) and
872 * a set of key/value attributes. Objects may be versioned. If a versioned object
873 * is written to, a new object with the same name but a different version is created,
874 * and the old version of the object is still accessible. If an unversioned object
875 * is written to, it is replaced, and the old data is not accessible.
881 * @brief Read operation on an Object
883 * This represents a Read operation on an Object. Read operations are optionally
884 * asynchronous, using the iterate() API.
888 const ceph::real_time
* mod_ptr
{nullptr};
889 const ceph::real_time
* unmod_ptr
{nullptr};
890 bool high_precision_time
{false};
891 uint32_t mod_zone_id
{0};
892 uint64_t mod_pg_ver
{0};
893 const char* if_match
{nullptr};
894 const char* if_nomatch
{nullptr};
895 ceph::real_time
* lastmod
{nullptr};
896 rgw_obj
* target_obj
{nullptr}; // XXX dang remove?
899 virtual ~ReadOp() = default;
901 /** Prepare the Read op. Must be called first */
902 virtual int prepare(optional_yield y
, const DoutPrefixProvider
* dpp
) = 0;
904 /** Synchronous read. Read from @a ofs to @a end (inclusive)
905 * into @a bl. Length is `end - ofs + 1`. */
906 virtual int read(int64_t ofs
, int64_t end
, bufferlist
& bl
,
907 optional_yield y
, const DoutPrefixProvider
* dpp
) = 0;
909 /** Asynchronous read. Read from @a ofs to @a end (inclusive)
910 * calling @a cb on each read chunk. Length is `end - ofs +
912 virtual int iterate(const DoutPrefixProvider
* dpp
, int64_t ofs
,
913 int64_t end
, RGWGetDataCB
* cb
, optional_yield y
) = 0;
915 /** Get an attribute by name */
916 virtual int get_attr(const DoutPrefixProvider
* dpp
, const char* name
, bufferlist
& dest
, optional_yield y
) = 0;
920 * @brief Delete operation on an Object
922 * This deletes an Object from the backing store.
926 ACLOwner bucket_owner
;
928 int versioning_status
{0};
929 uint64_t olh_epoch
{0};
930 std::string marker_version_id
;
931 uint32_t bilog_flags
{0};
932 std::list
<rgw_obj_index_key
>* remove_objs
{nullptr};
933 ceph::real_time expiration_time
;
934 ceph::real_time unmod_since
;
935 ceph::real_time mtime
;
936 bool high_precision_time
{false};
937 rgw_zone_set
* zones_trace
{nullptr};
939 uint64_t parts_accounted_size
{0};
943 bool delete_marker
{false};
944 std::string version_id
;
947 virtual ~DeleteOp() = default;
949 /** Delete the object */
950 virtual int delete_obj(const DoutPrefixProvider
* dpp
, optional_yield y
) = 0;
954 virtual ~Object() = default;
956 /** Shortcut synchronous delete call for common deletes */
957 virtual int delete_object(const DoutPrefixProvider
* dpp
,
959 bool prevent_versioning
= false) = 0;
960 /** Asynchronous delete call */
961 virtual int delete_obj_aio(const DoutPrefixProvider
* dpp
, RGWObjState
* astate
, Completions
* aio
,
962 bool keep_index_consistent
, optional_yield y
) = 0;
963 /** Copy an this object to another object. */
964 virtual int copy_object(User
* user
,
965 req_info
* info
, const rgw_zone_id
& source_zone
,
966 rgw::sal::Object
* dest_object
, rgw::sal::Bucket
* dest_bucket
,
967 rgw::sal::Bucket
* src_bucket
,
968 const rgw_placement_rule
& dest_placement
,
969 ceph::real_time
* src_mtime
, ceph::real_time
* mtime
,
970 const ceph::real_time
* mod_ptr
, const ceph::real_time
* unmod_ptr
,
971 bool high_precision_time
,
972 const char* if_match
, const char* if_nomatch
,
973 AttrsMod attrs_mod
, bool copy_if_newer
, Attrs
& attrs
,
974 RGWObjCategory category
, uint64_t olh_epoch
,
975 boost::optional
<ceph::real_time
> delete_at
,
976 std::string
* version_id
, std::string
* tag
, std::string
* etag
,
977 void (*progress_cb
)(off_t
, void *), void* progress_data
,
978 const DoutPrefixProvider
* dpp
, optional_yield y
) = 0;
979 /** Get the ACL for this object */
980 virtual RGWAccessControlPolicy
& get_acl(void) = 0;
981 /** Set the ACL for this object */
982 virtual int set_acl(const RGWAccessControlPolicy
& acl
) = 0;
983 /** Mark further operations on this object as being atomic */
984 virtual void set_atomic() = 0;
985 /** Check if this object is atomic */
986 virtual bool is_atomic() = 0;
987 /** Pre-fetch data when reading */
988 virtual void set_prefetch_data() = 0;
989 /** Check if this object should prefetch */
990 virtual bool is_prefetch_data() = 0;
991 /** Mark data as compressed */
992 virtual void set_compressed() = 0;
993 /** Check if this object is compressed */
994 virtual bool is_compressed() = 0;
995 /** Invalidate cached info about this object, except atomic, prefetch, and
997 virtual void invalidate() = 0;
999 /** Check to see if this object has an empty key. This means it's uninitialized */
1000 virtual bool empty() const = 0;
1001 /** Get the name of this object */
1002 virtual const std::string
&get_name() const = 0;
1004 /** Get the object state for this object. Will be removed in the future */
1005 virtual int get_obj_state(const DoutPrefixProvider
* dpp
, RGWObjState
**state
, optional_yield y
, bool follow_olh
= true) = 0;
1006 /** Set attributes for this object from the backing store. Attrs can be set or
1007 * deleted. @note the attribute APIs may be revisited in the future. */
1008 virtual int set_obj_attrs(const DoutPrefixProvider
* dpp
, Attrs
* setattrs
, Attrs
* delattrs
, optional_yield y
) = 0;
1009 /** Get attributes for this object */
1010 virtual int get_obj_attrs(optional_yield y
, const DoutPrefixProvider
* dpp
, rgw_obj
* target_obj
= NULL
) = 0;
1011 /** Modify attributes for this object. */
1012 virtual int modify_obj_attrs(const char* attr_name
, bufferlist
& attr_val
, optional_yield y
, const DoutPrefixProvider
* dpp
) = 0;
1013 /** Delete attributes for this object */
1014 virtual int delete_obj_attrs(const DoutPrefixProvider
* dpp
, const char* attr_name
, optional_yield y
) = 0;
1015 /** Check to see if this object has expired */
1016 virtual bool is_expired() = 0;
1017 /** Create a randomized instance ID for this object */
1018 virtual void gen_rand_obj_instance_name() = 0;
1019 /** Get a multipart serializer for this object */
1020 virtual std::unique_ptr
<MPSerializer
> get_serializer(const DoutPrefixProvider
*dpp
,
1021 const std::string
& lock_name
) = 0;
1022 /** Move the data of an object to new placement storage */
1023 virtual int transition(Bucket
* bucket
,
1024 const rgw_placement_rule
& placement_rule
,
1025 const real_time
& mtime
,
1027 const DoutPrefixProvider
* dpp
,
1028 optional_yield y
) = 0;
1029 /** Move an object to the cloud */
1030 virtual int transition_to_cloud(Bucket
* bucket
,
1031 rgw::sal::PlacementTier
* tier
,
1032 rgw_bucket_dir_entry
& o
,
1033 std::set
<std::string
>& cloud_targets
,
1036 const DoutPrefixProvider
* dpp
,
1037 optional_yield y
) = 0;
1038 /** Check to see if two placement rules match */
1039 virtual bool placement_rules_match(rgw_placement_rule
& r1
, rgw_placement_rule
& r2
) = 0;
1040 /** Dump driver-specific object layout info in JSON */
1041 virtual int dump_obj_layout(const DoutPrefixProvider
*dpp
, optional_yield y
, Formatter
* f
) = 0;
1043 /** Get the cached attributes for this object */
1044 virtual Attrs
& get_attrs(void) = 0;
1045 /** Get the (const) cached attributes for this object */
1046 virtual const Attrs
& get_attrs(void) const = 0;
1047 /** Set the cached attributes for this object */
1048 virtual int set_attrs(Attrs a
) = 0;
1049 /** Check to see if attributes are cached on this object */
1050 virtual bool has_attrs(void) = 0;
1051 /** Get the cached modification time for this object */
1052 virtual ceph::real_time
get_mtime(void) const = 0;
1053 /** Get the cached size for this object */
1054 virtual uint64_t get_obj_size(void) const = 0;
1055 /** Get the bucket containing this object */
1056 virtual Bucket
* get_bucket(void) const = 0;
1057 /** Set the bucket containing this object */
1058 virtual void set_bucket(Bucket
* b
) = 0;
1059 /** Get the sharding hash representation of this object */
1060 virtual std::string
get_hash_source(void) = 0;
1061 /** Set the sharding hash representation of this object */
1062 virtual void set_hash_source(std::string s
) = 0;
1063 /** Build an Object Identifier string for this object */
1064 virtual std::string
get_oid(void) const = 0;
1065 /** True if this object is a delete marker (newest version is deleted) */
1066 virtual bool get_delete_marker(void) = 0;
1067 /** True if this object is stored in the extra data pool */
1068 virtual bool get_in_extra_data(void) = 0;
1069 /** Set the in_extra_data field */
1070 virtual void set_in_extra_data(bool i
) = 0;
1071 /** Helper to sanitize object size, offset, and end values */
1072 int range_to_ofs(uint64_t obj_size
, int64_t &ofs
, int64_t &end
);
1073 /** Set the cached size of this object */
1074 virtual void set_obj_size(uint64_t s
) = 0;
1075 /** Set the cached name of this object */
1076 virtual void set_name(const std::string
& n
) = 0;
1077 /** Set the cached key of this object */
1078 virtual void set_key(const rgw_obj_key
& k
) = 0;
1079 /** Get an rgw_obj representing this object */
1080 virtual rgw_obj
get_obj(void) const = 0;
1082 /** Restore the previous swift version of this object */
1083 virtual int swift_versioning_restore(bool& restored
, /* out */
1084 const DoutPrefixProvider
* dpp
) = 0;
1085 /** Copy the current version of a swift object to the configured destination bucket*/
1086 virtual int swift_versioning_copy(const DoutPrefixProvider
* dpp
,
1087 optional_yield y
) = 0;
1089 /** Get a new ReadOp for this object */
1090 virtual std::unique_ptr
<ReadOp
> get_read_op() = 0;
1091 /** Get a new DeleteOp for this object */
1092 virtual std::unique_ptr
<DeleteOp
> get_delete_op() = 0;
1094 /** Get @a count OMAP values via listing, starting at @a marker for this object */
1095 virtual int omap_get_vals(const DoutPrefixProvider
*dpp
, const std::string
& marker
, uint64_t count
,
1096 std::map
<std::string
, bufferlist
>* m
,
1097 bool* pmore
, optional_yield y
) = 0;
1098 /** Get all OMAP key/value pairs for this object */
1099 virtual int omap_get_all(const DoutPrefixProvider
*dpp
, std::map
<std::string
, bufferlist
>* m
,
1100 optional_yield y
) = 0;
1101 /** Get the OMAP values matching the given set of keys */
1102 virtual int omap_get_vals_by_keys(const DoutPrefixProvider
*dpp
, const std::string
& oid
,
1103 const std::set
<std::string
>& keys
,
1105 /** Get a single OMAP value matching the given key */
1106 virtual int omap_set_val_by_key(const DoutPrefixProvider
*dpp
, const std::string
& key
, bufferlist
& val
,
1107 bool must_exist
, optional_yield y
) = 0;
1108 /** Change the ownership of this object */
1109 virtual int chown(User
& new_user
, const DoutPrefixProvider
* dpp
, optional_yield y
) = 0;
1111 /** Check to see if the given object pointer is uninitialized */
1112 static bool empty(const Object
* o
) { return (!o
|| o
->empty()); }
1113 /** Check to see if the given object unique pointer is uninitialized */
1114 static bool empty(const std::unique_ptr
<Object
>& o
) { return (!o
|| o
->empty()); }
1115 /** Get a unique copy of this object */
1116 virtual std::unique_ptr
<Object
> clone() = 0;
1118 /* dang - This is temporary, until the API is completed */
1119 /** Get the key for this object */
1120 virtual rgw_obj_key
& get_key() = 0;
1121 /** Set the instance for this object */
1122 virtual void set_instance(const std::string
&i
) = 0;
1123 /** Get the instance for this object */
1124 virtual const std::string
&get_instance() const = 0;
1125 /** Check to see if this object has an instance set */
1126 virtual bool have_instance(void) = 0;
1127 /** Clear the instance on this object */
1128 virtual void clear_instance() = 0;
1130 /** Print the User to @a out */
1131 virtual void print(std::ostream
& out
) const = 0;
1133 friend inline std::ostream
& operator<<(std::ostream
& out
, const Object
& o
) {
1137 friend inline std::ostream
& operator<<(std::ostream
& out
, const Object
* o
) {
1144 friend inline std::ostream
& operator<<(std::ostream
& out
, const std::unique_ptr
<Object
>& p
) {
1151 * @brief Abstraction of a single part of a multipart upload
1153 class MultipartPart
{
1155 MultipartPart() = default;
1156 virtual ~MultipartPart() = default;
1158 /** Get the part number of this part */
1159 virtual uint32_t get_num() = 0;
1160 /** Get the size of this part */
1161 virtual uint64_t get_size() = 0;
1162 /** Get the etag of this part */
1163 virtual const std::string
& get_etag() = 0;
1164 /** Get the modification time of this part */
1165 virtual ceph::real_time
& get_mtime() = 0;
1169 * @brief Abstraction of a multipart upload
1171 * This represents a multipart upload. For large objects, it's inefficient to do a
1172 * single, long-lived upload of the object. Instead, protocols such as S3 allow the
1173 * client to start a multipart upload, and then upload object in smaller parts in
1174 * parallel. A MultipartUpload consists of a target bucket, a unique identifier, and a
1175 * set of upload parts.
1177 class MultipartUpload
{
1179 MultipartUpload() = default;
1180 virtual ~MultipartUpload() = default;
1182 /** Get the name of the object representing this upload in the backing store */
1183 virtual const std::string
& get_meta() const = 0;
1184 /** Get the name of the target object for this upload */
1185 virtual const std::string
& get_key() const = 0;
1186 /** Get the unique ID of this upload */
1187 virtual const std::string
& get_upload_id() const = 0;
1188 /** Get the owner of this upload */
1189 virtual const ACLOwner
& get_owner() const = 0;
1190 /** Get the modification time of this upload */
1191 virtual ceph::real_time
& get_mtime() = 0;
1193 /** Get all the cached parts that make up this upload */
1194 virtual std::map
<uint32_t, std::unique_ptr
<MultipartPart
>>& get_parts() = 0;
1196 /** Get the trace context of this upload */
1197 virtual const jspan_context
& get_trace() = 0;
1199 /** Get the Object that represents this upload */
1200 virtual std::unique_ptr
<rgw::sal::Object
> get_meta_obj() = 0;
1202 /** Initialize this upload */
1203 virtual int init(const DoutPrefixProvider
* dpp
, optional_yield y
, ACLOwner
& owner
, rgw_placement_rule
& dest_placement
, rgw::sal::Attrs
& attrs
) = 0;
1204 /** List all the parts of this upload, filling the parts cache */
1205 virtual int list_parts(const DoutPrefixProvider
* dpp
, CephContext
* cct
,
1206 int num_parts
, int marker
,
1207 int* next_marker
, bool* truncated
,
1208 bool assume_unsorted
= false) = 0;
1209 /** Abort this upload */
1210 virtual int abort(const DoutPrefixProvider
* dpp
, CephContext
* cct
) = 0;
1211 /** Complete this upload, making it available as a normal object */
1212 virtual int complete(const DoutPrefixProvider
* dpp
,
1213 optional_yield y
, CephContext
* cct
,
1214 std::map
<int, std::string
>& part_etags
,
1215 std::list
<rgw_obj_index_key
>& remove_objs
,
1216 uint64_t& accounted_size
, bool& compressed
,
1217 RGWCompressionInfo
& cs_info
, off_t
& ofs
,
1218 std::string
& tag
, ACLOwner
& owner
,
1220 rgw::sal::Object
* target_obj
) = 0;
1222 /** Get placement and/or attribute info for this upload */
1223 virtual int get_info(const DoutPrefixProvider
*dpp
, optional_yield y
, rgw_placement_rule
** rule
, rgw::sal::Attrs
* attrs
= nullptr) = 0;
1225 /** Get a Writer to write to a part of this upload */
1226 virtual std::unique_ptr
<Writer
> get_writer(const DoutPrefixProvider
*dpp
,
1228 rgw::sal::Object
* obj
,
1229 const rgw_user
& owner
,
1230 const rgw_placement_rule
*ptail_placement_rule
,
1232 const std::string
& part_num_str
) = 0;
1234 /** Print the Upload to @a out */
1235 virtual void print(std::ostream
& out
) const = 0;
1237 friend inline std::ostream
& operator<<(std::ostream
& out
, const MultipartUpload
& u
) {
1241 friend inline std::ostream
& operator<<(std::ostream
& out
, const MultipartUpload
* u
) {
1248 friend inline std::ostream
& operator<<(std::ostream
& out
, const
1249 std::unique_ptr
<MultipartUpload
>& p
) {
1256 * @brief Interface of a lock/serialization
1260 Serializer() = default;
1261 virtual ~Serializer() = default;
1263 /** Try to take the lock for the given amount of time. */
1264 virtual int try_lock(const DoutPrefixProvider
*dpp
, utime_t dur
, optional_yield y
) = 0;
1265 /** Unlock the lock */
1266 virtual int unlock() = 0;
1268 /** Print the Serializer to @a out */
1269 virtual void print(std::ostream
& out
) const = 0;
1271 friend inline std::ostream
& operator<<(std::ostream
& out
, const Serializer
& s
) {
1275 friend inline std::ostream
& operator<<(std::ostream
& out
, const Serializer
* s
) {
1284 /** @brief Abstraction of a serializer for multipart uploads
1286 class MPSerializer
: public Serializer
{
1288 MPSerializer() = default;
1289 virtual ~MPSerializer() = default;
1291 virtual void clear_locked() = 0;
1292 /** Check to see if locked */
1293 virtual bool is_locked() = 0;
1296 /** @brief Abstraction of a serializer for Lifecycle
1298 class LCSerializer
: public Serializer
{
1301 virtual ~LCSerializer() = default;
1305 * @brief Abstraction for lifecycle processing
1307 * Lifecycle processing loops over the objects in a bucket, applying per-bucket policy
1308 * to each object. Examples of policy can be deleting after a certain amount of time,
1309 * deleting extra versions, changing the storage class, and so on.
1313 /** Head of a lifecycle run. Used for tracking parallel lifecycle runs. */
1316 virtual ~LCHead() = default;
1318 virtual time_t& get_start_date() = 0;
1319 virtual void set_start_date(time_t) = 0;
1320 virtual std::string
& get_marker() = 0;
1321 virtual void set_marker(const std::string
&) = 0;
1322 virtual time_t& get_shard_rollover_date() = 0;
1323 virtual void set_shard_rollover_date(time_t) = 0;
1326 /** Single entry in a lifecycle run. Multiple entries can exist processing different
1329 LCEntry() = default;
1330 virtual ~LCEntry() = default;
1332 virtual std::string
& get_bucket() = 0;
1333 virtual void set_bucket(const std::string
&) = 0;
1334 virtual std::string
& get_oid() = 0;
1335 virtual void set_oid(const std::string
&) = 0;
1336 virtual uint64_t get_start_time() = 0;
1337 virtual void set_start_time(uint64_t) = 0;
1338 virtual uint32_t get_status() = 0;
1339 virtual void set_status(uint32_t) = 0;
1341 /** Print the entry to @a out */
1342 virtual void print(std::ostream
& out
) const = 0;
1344 friend inline std::ostream
& operator<<(std::ostream
& out
, const LCEntry
& e
) {
1348 friend inline std::ostream
& operator<<(std::ostream
& out
, const LCEntry
* e
) {
1355 friend inline std::ostream
& operator<<(std::ostream
& out
, const std::unique_ptr
<LCEntry
>& p
) {
1361 Lifecycle() = default;
1362 virtual ~Lifecycle() = default;
1364 /** Get an empty entry */
1365 virtual std::unique_ptr
<LCEntry
> get_entry() = 0;
1366 /** Get an entry matching the given marker */
1367 virtual int get_entry(const std::string
& oid
, const std::string
& marker
, std::unique_ptr
<LCEntry
>* entry
) = 0;
1368 /** Get the entry following the given marker */
1369 virtual int get_next_entry(const std::string
& oid
, const std::string
& marker
, std::unique_ptr
<LCEntry
>* entry
) = 0;
1370 /** Store a modified entry in then backing store */
1371 virtual int set_entry(const std::string
& oid
, LCEntry
& entry
) = 0;
1372 /** List all known entries */
1373 virtual int list_entries(const std::string
& oid
, const std::string
& marker
,
1374 uint32_t max_entries
,
1375 std::vector
<std::unique_ptr
<LCEntry
>>& entries
) = 0;
1376 /** Remove an entry from the backing store */
1377 virtual int rm_entry(const std::string
& oid
, LCEntry
& entry
) = 0;
1379 virtual int get_head(const std::string
& oid
, std::unique_ptr
<LCHead
>* head
) = 0;
1380 /** Store a modified head to the backing store */
1381 virtual int put_head(const std::string
& oid
, LCHead
& head
) = 0;
1383 /** Get a serializer for lifecycle */
1384 virtual std::unique_ptr
<LCSerializer
> get_serializer(const std::string
& lock_name
,
1385 const std::string
& oid
,
1386 const std::string
& cookie
) = 0;
1390 * @brief Abstraction for a Notification event
1392 * RGW can generate notifications for various events, such as object creation or
1395 class Notification
{
1400 virtual ~Notification() = default;
1402 /** Indicate the start of the event associated with this notification */
1403 virtual int publish_reserve(const DoutPrefixProvider
*dpp
, RGWObjTags
* obj_tags
= nullptr) = 0;
1404 /** Indicate the successful completion of the event associated with this notification */
1405 virtual int publish_commit(const DoutPrefixProvider
* dpp
, uint64_t size
,
1406 const ceph::real_time
& mtime
, const std::string
& etag
, const std::string
& version
) = 0;
1410 * @brief Abstraction for an asynchronous writer
1412 * Writing is done through a set of filters. This allows chaining filters to do things
1413 * like compression and encryption on async writes. This is the base abstraction for
1416 class Writer
: public ObjectProcessor
{
1419 virtual ~Writer() = default;
1421 /** prepare to start processing object data */
1422 virtual int prepare(optional_yield y
) = 0;
1425 * Process a buffer. Called multiple times to write different buffers.
1426 * data.length() == 0 indicates the last call and may be used to flush
1429 virtual int process(bufferlist
&& data
, uint64_t offset
) = 0;
1431 /** complete the operation and make its result visible to clients */
1432 virtual int complete(size_t accounted_size
, const std::string
& etag
,
1433 ceph::real_time
*mtime
, ceph::real_time set_mtime
,
1434 std::map
<std::string
, bufferlist
>& attrs
,
1435 ceph::real_time delete_at
,
1436 const char *if_match
, const char *if_nomatch
,
1437 const std::string
*user_data
,
1438 rgw_zone_set
*zones_trace
, bool *canceled
,
1439 optional_yield y
) = 0;
1444 * @brief Abstraction of a placement tier
1446 * This abstraction allows access to information about placement tiers,
1447 * including storage class.
1449 class PlacementTier
{
1451 virtual ~PlacementTier() = default;
1453 /** Get the type of this tier */
1454 virtual const std::string
& get_tier_type() = 0;
1455 /** Get the storage class of this tier */
1456 virtual const std::string
& get_storage_class() = 0;
1457 /** Should we retain the head object when transitioning */
1458 virtual bool retain_head_object() = 0;
1459 /** Get the placement rule associated with this tier */
1463 * @brief Abstraction of a zone group
1465 * This class allows access to information about a zonegroup. It may be the
1466 * group containing the current zone, or another group.
1470 virtual ~ZoneGroup() = default;
1471 /** Get the ID of this zonegroup */
1472 virtual const std::string
& get_id() const = 0;
1473 /** Get the name of this zonegroup */
1474 virtual const std::string
& get_name() const = 0;
1475 /** Determine if two zonegroups are the same */
1476 virtual int equals(const std::string
& other_zonegroup
) const = 0;
1477 /** Get the endpoint from zonegroup, or from master zone if not set */
1478 virtual const std::string
& get_endpoint() const = 0;
1479 /** Check if a placement target (by name) exists in this zonegroup */
1480 virtual bool placement_target_exists(std::string
& target
) const = 0;
1481 /** Check if this is the master zonegroup */
1482 virtual bool is_master_zonegroup() const = 0;
1483 /** Get the API name of this zonegroup */
1484 virtual const std::string
& get_api_name() const = 0;
1485 /** Get the list of placement target names for this zone */
1486 virtual int get_placement_target_names(std::set
<std::string
>& names
) const = 0;
1487 /** Get the name of the default placement target for this zone */
1488 virtual const std::string
& get_default_placement_name() const = 0;
1489 /** Get the list of hostnames from this zone */
1490 virtual int get_hostnames(std::list
<std::string
>& names
) const = 0;
1491 /** Get the list of hostnames that host s3 websites from this zone */
1492 virtual int get_s3website_hostnames(std::list
<std::string
>& names
) const = 0;
1493 /** Get the number of zones in this zonegroup */
1494 virtual int get_zone_count() const = 0;
1495 /** Get the placement tier associated with the rule */
1496 virtual int get_placement_tier(const rgw_placement_rule
& rule
, std::unique_ptr
<PlacementTier
>* tier
) = 0;
1497 /** Get a zone by ID */
1498 virtual int get_zone_by_id(const std::string
& id
, std::unique_ptr
<Zone
>* zone
) = 0;
1499 /** Get a zone by Name */
1500 virtual int get_zone_by_name(const std::string
& name
, std::unique_ptr
<Zone
>* zone
) = 0;
1501 /** List zones in zone group by ID */
1502 virtual int list_zones(std::list
<std::string
>& zone_ids
) = 0;
1503 /// Return true if the given feature is enabled in the zonegroup.
1504 virtual bool supports(std::string_view feature
) const = 0;
1505 /** Clone a copy of this zonegroup. */
1506 virtual std::unique_ptr
<ZoneGroup
> clone() = 0;
1510 * @brief Abstraction of a Zone
1512 * This abstraction allows access to information about zones. This can be the zone
1513 * containing the RGW, or another zone.
1517 virtual ~Zone() = default;
1519 /** Clone a copy of this zone. */
1520 virtual std::unique_ptr
<Zone
> clone() = 0;
1521 /** Get info about the zonegroup containing this zone */
1522 virtual ZoneGroup
& get_zonegroup() = 0;
1523 /** Get the ID of this zone */
1524 virtual const std::string
& get_id() = 0;
1525 /** Get the name of this zone */
1526 virtual const std::string
& get_name() const = 0;
1527 /** True if this zone is writable */
1528 virtual bool is_writeable() = 0;
1529 /** Get the URL for the endpoint for redirecting to this zone */
1530 virtual bool get_redirect_endpoint(std::string
* endpoint
) = 0;
1531 /** Check to see if the given API is supported in this zone */
1532 virtual bool has_zonegroup_api(const std::string
& api
) const = 0;
1533 /** Get the current period ID for this zone */
1534 virtual const std::string
& get_current_period_id() = 0;
1535 /** Get thes system access key for this zone */
1536 virtual const RGWAccessKey
& get_system_key() = 0;
1537 /** Get the name of the realm containing this zone */
1538 virtual const std::string
& get_realm_name() = 0;
1539 /** Get the ID of the realm containing this zone */
1540 virtual const std::string
& get_realm_id() = 0;
1541 /** Get the tier type for the zone */
1542 virtual const std::string_view
get_tier_type() = 0;
1543 /** Get a handler for zone sync policy. */
1544 virtual RGWBucketSyncPolicyHandlerRef
get_sync_policy_handler() = 0;
1548 * @brief Abstraction of a manager for Lua scripts and packages
1550 * RGW can load and process Lua scripts. This will handle loading/storing scripts; adding, deleting, and listing packages
1554 virtual ~LuaManager() = default;
1556 /** Get a script named with the given key from the backing store */
1557 virtual int get_script(const DoutPrefixProvider
* dpp
, optional_yield y
, const std::string
& key
, std::string
& script
) = 0;
1558 /** Put a script named with the given key to the backing store */
1559 virtual int put_script(const DoutPrefixProvider
* dpp
, optional_yield y
, const std::string
& key
, const std::string
& script
) = 0;
1560 /** Delete a script named with the given key from the backing store */
1561 virtual int del_script(const DoutPrefixProvider
* dpp
, optional_yield y
, const std::string
& key
) = 0;
1562 /** Add a lua package */
1563 virtual int add_package(const DoutPrefixProvider
* dpp
, optional_yield y
, const std::string
& package_name
) = 0;
1564 /** Remove a lua package */
1565 virtual int remove_package(const DoutPrefixProvider
* dpp
, optional_yield y
, const std::string
& package_name
) = 0;
1566 /** List lua packages */
1567 virtual int list_packages(const DoutPrefixProvider
* dpp
, optional_yield y
, rgw::lua::packages_t
& packages
) = 0;
1570 /** @} namespace rgw::sal in group RGWSAL */
1571 } } // namespace rgw::sal
1574 * @brief A manager for Drivers
1576 * This will manage the singleton instances of the various drivers. Drivers come in two
1577 * varieties: Full and Raw. A full driver is suitable for use in a radosgw daemon. It
1578 * has full access to the cluster, if any. A raw driver is a stripped down driver, used
1579 * for admin commands.
1581 class DriverManager
{
1584 /** Name of store to create */
1585 std::string store_name
;
1586 /** Name of filter to create or "none" */
1587 std::string filter_name
;
1591 /** Get a full driver by service name */
1592 static rgw::sal::Driver
* get_storage(const DoutPrefixProvider
* dpp
,
1598 bool run_sync_thread
,
1599 bool run_reshard_thread
,
1600 bool use_cache
= true,
1601 bool use_gc
= true) {
1602 rgw::sal::Driver
* driver
= init_storage_provider(dpp
, cct
, cfg
, use_gc_thread
,
1610 /** Get a stripped down driver by service name */
1611 static rgw::sal::Driver
* get_raw_storage(const DoutPrefixProvider
* dpp
,
1612 CephContext
* cct
, const Config
& cfg
) {
1613 rgw::sal::Driver
* driver
= init_raw_storage_provider(dpp
, cct
, cfg
);
1616 /** Initialize a new full Driver */
1617 static rgw::sal::Driver
* init_storage_provider(const DoutPrefixProvider
* dpp
,
1623 bool run_sync_thread
,
1624 bool run_reshard_thread
,
1625 bool use_metadata_cache
,
1627 /** Initialize a new raw Driver */
1628 static rgw::sal::Driver
* init_raw_storage_provider(const DoutPrefixProvider
* dpp
,
1631 /** Close a Driver when it's no longer needed */
1632 static void close_storage(rgw::sal::Driver
* driver
);
1634 /** Get the config for Drivers */
1635 static Config
get_config(bool admin
, CephContext
* cct
);
1637 /** Create a ConfigStore */
1638 static auto create_config_store(const DoutPrefixProvider
* dpp
,
1639 std::string_view type
)
1640 -> std::unique_ptr
<rgw::sal::ConfigStore
>;