1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 #ifndef CEPH_RGW_BUCKET_H
5 #define CEPH_RGW_BUCKET_H
10 #include "include/types.h"
11 #include "rgw_common.h"
12 #include "rgw_tools.h"
14 #include "rgw_rados.h"
16 #include "rgw_string.h"
18 #include "common/Formatter.h"
19 #include "common/lru_map.h"
20 #include "common/ceph_time.h"
21 #include "rgw_formats.h"
23 // define as static when RGWBucket implementation compete
24 extern void rgw_get_buckets_obj(const rgw_user
& user_id
, string
& buckets_obj_id
);
26 extern int rgw_bucket_store_info(RGWRados
*store
, const string
& bucket_name
, bufferlist
& bl
, bool exclusive
,
27 map
<string
, bufferlist
> *pattrs
, RGWObjVersionTracker
*objv_tracker
,
29 extern int rgw_bucket_instance_store_info(RGWRados
*store
, string
& oid
, bufferlist
& bl
, bool exclusive
,
30 map
<string
, bufferlist
> *pattrs
, RGWObjVersionTracker
*objv_tracker
,
33 extern int rgw_bucket_parse_bucket_instance(const string
& bucket_instance
, string
*target_bucket_instance
, int *shard_id
);
34 extern int rgw_bucket_parse_bucket_key(CephContext
*cct
, const string
& key
,
35 rgw_bucket
* bucket
, int *shard_id
);
37 extern int rgw_bucket_instance_remove_entry(RGWRados
*store
, string
& entry
, RGWObjVersionTracker
*objv_tracker
);
38 extern void rgw_bucket_instance_key_to_oid(string
& key
);
39 extern void rgw_bucket_instance_oid_to_key(string
& oid
);
41 extern int rgw_bucket_delete_bucket_obj(RGWRados
*store
,
42 const string
& tenant_name
,
43 const string
& bucket_name
,
44 RGWObjVersionTracker
& objv_tracker
);
46 extern int rgw_bucket_sync_user_stats(RGWRados
*store
, const rgw_user
& user_id
, const RGWBucketInfo
& bucket_info
);
47 extern int rgw_bucket_sync_user_stats(RGWRados
*store
, const string
& tenant_name
, const string
& bucket_name
);
49 extern std::string
rgw_make_bucket_entry_name(const std::string
& tenant_name
,
50 const std::string
& bucket_name
);
51 static inline void rgw_make_bucket_entry_name(const string
& tenant_name
,
52 const string
& bucket_name
,
53 std::string
& bucket_entry
) {
54 bucket_entry
= rgw_make_bucket_entry_name(tenant_name
, bucket_name
);
57 extern void rgw_parse_url_bucket(const string
& bucket
,
58 const string
& auth_tenant
,
59 string
&tenant_name
, string
&bucket_name
);
61 struct RGWBucketCompleteInfo
{
63 map
<string
, bufferlist
> attrs
;
65 void dump(Formatter
*f
) const;
66 void decode_json(JSONObj
*obj
);
69 class RGWBucketEntryMetadataObject
: public RGWMetadataObject
{
70 RGWBucketEntryPoint ep
;
72 RGWBucketEntryMetadataObject(RGWBucketEntryPoint
& _ep
, obj_version
& v
, real_time m
) : ep(_ep
) {
77 void dump(Formatter
*f
) const override
{
82 class RGWBucketInstanceMetadataObject
: public RGWMetadataObject
{
83 RGWBucketCompleteInfo info
;
85 RGWBucketInstanceMetadataObject() {}
86 RGWBucketInstanceMetadataObject(RGWBucketCompleteInfo
& i
, obj_version
& v
, real_time m
) : info(i
) {
91 void dump(Formatter
*f
) const override
{
95 void decode_json(JSONObj
*obj
) {
96 info
.decode_json(obj
);
99 RGWBucketInfo
& get_bucket_info() { return info
.info
; }
103 * Store a list of the user's buckets, with associated functinos.
107 map
<string
, RGWBucketEnt
> buckets
;
111 void encode(bufferlist
& bl
) const {
112 ::encode(buckets
, bl
);
114 void decode(bufferlist::iterator
& bl
) {
115 ::decode(buckets
, bl
);
118 * Check if the user owns a bucket by the given name.
120 bool owns(string
& name
) {
121 map
<string
, RGWBucketEnt
>::iterator iter
;
122 iter
= buckets
.find(name
);
123 return (iter
!= buckets
.end());
127 * Add a (created) bucket to the user's bucket list.
129 void add(const RGWBucketEnt
& bucket
) {
130 buckets
[bucket
.bucket
.name
] = bucket
;
134 * Remove a bucket from the user's list by name.
136 void remove(string
& name
) {
137 map
<string
, RGWBucketEnt
>::iterator iter
;
138 iter
= buckets
.find(name
);
139 if (iter
!= buckets
.end()) {
145 * Get the user's buckets as a map.
147 map
<string
, RGWBucketEnt
>& get_buckets() { return buckets
; }
150 * Cleanup data structure
152 void clear() { buckets
.clear(); }
154 size_t count() { return buckets
.size(); }
156 WRITE_CLASS_ENCODER(RGWUserBuckets
)
158 class RGWMetadataManager
;
160 extern void rgw_bucket_init(RGWMetadataManager
*mm
);
162 * Get all the buckets owned by a user and fill up an RGWUserBuckets with them.
163 * Returns: 0 on success, -ERR# on failure.
165 extern int rgw_read_user_buckets(RGWRados
*store
,
166 const rgw_user
& user_id
,
167 RGWUserBuckets
& buckets
,
168 const string
& marker
,
169 const string
& end_marker
,
173 uint64_t default_amount
= 1000);
175 extern int rgw_link_bucket(RGWRados
*store
, const rgw_user
& user_id
, rgw_bucket
& bucket
, real_time creation_time
, bool update_entrypoint
= true);
176 extern int rgw_unlink_bucket(RGWRados
*store
, const rgw_user
& user_id
,
177 const string
& tenant_name
, const string
& bucket_name
, bool update_entrypoint
= true);
179 extern int rgw_remove_object(RGWRados
*store
, RGWBucketInfo
& bucket_info
, rgw_bucket
& bucket
, rgw_obj_key
& key
);
180 extern int rgw_remove_bucket(RGWRados
*store
, rgw_bucket
& bucket
, bool delete_children
);
181 extern int rgw_remove_bucket_bypass_gc(RGWRados
*store
, rgw_bucket
& bucket
, int concurrent_max
);
183 extern int rgw_bucket_set_attrs(RGWRados
*store
, RGWBucketInfo
& bucket_info
,
184 map
<string
, bufferlist
>& attrs
,
185 RGWObjVersionTracker
*objv_tracker
);
187 extern void check_bad_user_bucket_mapping(RGWRados
*store
, const rgw_user
& user_id
, bool fix
);
189 struct RGWBucketAdminOpState
{
191 std::string display_name
;
192 std::string bucket_name
;
193 std::string bucket_id
;
194 std::string object_name
;
200 bool delete_child_objects
;
206 void set_fetch_stats(bool value
) { stat_buckets
= value
; }
207 void set_check_objects(bool value
) { check_objects
= value
; }
208 void set_fix_index(bool value
) { fix_index
= value
; }
209 void set_delete_children(bool value
) { delete_child_objects
= value
; }
211 void set_max_aio(int value
) { max_aio
= value
; }
213 void set_user_id(const rgw_user
& user_id
) {
214 if (!user_id
.empty())
217 void set_bucket_name(const std::string
& bucket_str
) {
218 bucket_name
= bucket_str
;
220 void set_object(std::string
& object_str
) {
221 object_name
= object_str
;
224 rgw_user
& get_user_id() { return uid
; }
225 std::string
& get_user_display_name() { return display_name
; }
226 std::string
& get_bucket_name() { return bucket_name
; }
227 std::string
& get_object_name() { return object_name
; }
229 rgw_bucket
& get_bucket() { return bucket
; }
230 void set_bucket(rgw_bucket
& _bucket
) {
232 bucket_stored
= true;
235 void set_bucket_id(const string
& bi
) {
238 const string
& get_bucket_id() { return bucket_id
; }
240 bool will_fetch_stats() { return stat_buckets
; }
241 bool will_fix_index() { return fix_index
; }
242 bool will_delete_children() { return delete_child_objects
; }
243 bool will_check_objects() { return check_objects
; }
244 bool is_user_op() { return !uid
.empty(); }
245 bool is_system_op() { return uid
.empty(); }
246 bool has_bucket_stored() { return bucket_stored
; }
247 int get_max_aio() { return max_aio
; }
249 RGWBucketAdminOpState() : list_buckets(false), stat_buckets(false), check_objects(false),
250 fix_index(false), delete_child_objects(false),
251 bucket_stored(false) {}
255 * A simple wrapper class for administrative bucket operations
260 RGWUserBuckets buckets
;
262 RGWAccessHandle handle
;
264 RGWUserInfo user_info
;
266 std::string bucket_name
;
270 RGWBucketInfo bucket_info
;
273 RGWBucket() : store(NULL
), handle(NULL
), failure(false) {}
274 int init(RGWRados
*storage
, RGWBucketAdminOpState
& op_state
);
276 int check_bad_index_multipart(RGWBucketAdminOpState
& op_state
,
277 list
<rgw_obj_index_key
>& objs_to_unlink
, std::string
*err_msg
= NULL
);
279 int check_object_index(RGWBucketAdminOpState
& op_state
,
280 RGWFormatterFlusher
& flusher
,
281 std::string
*err_msg
= NULL
);
283 int check_index(RGWBucketAdminOpState
& op_state
,
284 map
<RGWObjCategory
, RGWStorageStats
>& existing_stats
,
285 map
<RGWObjCategory
, RGWStorageStats
>& calculated_stats
,
286 std::string
*err_msg
= NULL
);
288 int remove(RGWBucketAdminOpState
& op_state
, bool bypass_gc
= false, bool keep_index_consistent
= true, std::string
*err_msg
= NULL
);
289 int link(RGWBucketAdminOpState
& op_state
, std::string
*err_msg
= NULL
);
290 int unlink(RGWBucketAdminOpState
& op_state
, std::string
*err_msg
= NULL
);
292 int remove_object(RGWBucketAdminOpState
& op_state
, std::string
*err_msg
= NULL
);
293 int policy_bl_to_stream(bufferlist
& bl
, ostream
& o
);
294 int get_policy(RGWBucketAdminOpState
& op_state
, RGWAccessControlPolicy
& policy
);
296 void clear_failure() { failure
= false; }
299 class RGWBucketAdminOp
302 static int get_policy(RGWRados
*store
, RGWBucketAdminOpState
& op_state
,
303 RGWFormatterFlusher
& flusher
);
304 static int get_policy(RGWRados
*store
, RGWBucketAdminOpState
& op_state
,
305 RGWAccessControlPolicy
& policy
);
306 static int dump_s3_policy(RGWRados
*store
, RGWBucketAdminOpState
& op_state
,
309 static int unlink(RGWRados
*store
, RGWBucketAdminOpState
& op_state
);
310 static int link(RGWRados
*store
, RGWBucketAdminOpState
& op_state
, string
*err_msg
= NULL
);
312 static int check_index(RGWRados
*store
, RGWBucketAdminOpState
& op_state
,
313 RGWFormatterFlusher
& flusher
);
315 static int remove_bucket(RGWRados
*store
, RGWBucketAdminOpState
& op_state
, bool bypass_gc
= false, bool keep_index_consistent
= true);
316 static int remove_object(RGWRados
*store
, RGWBucketAdminOpState
& op_state
);
317 static int info(RGWRados
*store
, RGWBucketAdminOpState
& op_state
, RGWFormatterFlusher
& flusher
);
318 static int limit_check(RGWRados
*store
, RGWBucketAdminOpState
& op_state
,
319 const std::list
<std::string
>& user_ids
,
320 RGWFormatterFlusher
& flusher
,
321 bool warnings_only
= false);
325 enum DataLogEntityType
{
326 ENTITY_TYPE_UNKNOWN
= 0,
327 ENTITY_TYPE_BUCKET
= 1,
330 struct rgw_data_change
{
331 DataLogEntityType entity_type
;
335 void encode(bufferlist
& bl
) const {
336 ENCODE_START(1, 1, bl
);
337 uint8_t t
= (uint8_t)entity_type
;
340 ::encode(timestamp
, bl
);
344 void decode(bufferlist::iterator
& bl
) {
348 entity_type
= (DataLogEntityType
)t
;
350 ::decode(timestamp
, bl
);
354 void dump(Formatter
*f
) const;
355 void decode_json(JSONObj
*obj
);
357 WRITE_CLASS_ENCODER(rgw_data_change
)
359 struct rgw_data_change_log_entry
{
361 real_time log_timestamp
;
362 rgw_data_change entry
;
364 void encode(bufferlist
& bl
) const {
365 ENCODE_START(1, 1, bl
);
366 ::encode(log_id
, bl
);
367 ::encode(log_timestamp
, bl
);
372 void decode(bufferlist::iterator
& bl
) {
374 ::decode(log_id
, bl
);
375 ::decode(log_timestamp
, bl
);
380 void dump(Formatter
*f
) const;
381 void decode_json(JSONObj
*obj
);
383 WRITE_CLASS_ENCODER(rgw_data_change_log_entry
)
385 struct RGWDataChangesLogInfo
{
387 real_time last_update
;
389 void dump(Formatter
*f
) const;
390 void decode_json(JSONObj
*obj
);
393 class RGWDataChangesLog
{
401 RWLock modified_lock
;
402 map
<int, set
<string
> > modified_shards
;
404 std::atomic
<bool> down_flag
= { false };
406 struct ChangeStatus
{
407 real_time cur_expiration
;
410 RefCountedCond
*cond
;
413 ChangeStatus() : pending(false), cond(NULL
) {
414 lock
= new Mutex("RGWDataChangesLog::ChangeStatus");
422 typedef ceph::shared_ptr
<ChangeStatus
> ChangeStatusPtr
;
424 lru_map
<rgw_bucket_shard
, ChangeStatusPtr
> changes
;
426 map
<rgw_bucket_shard
, bool> cur_cycle
;
428 void _get_change(const rgw_bucket_shard
& bs
, ChangeStatusPtr
& status
);
429 void register_renew(rgw_bucket_shard
& bs
);
430 void update_renewed(rgw_bucket_shard
& bs
, real_time
& expiration
);
432 class ChangesRenewThread
: public Thread
{
434 RGWDataChangesLog
*log
;
439 ChangesRenewThread(CephContext
*_cct
, RGWDataChangesLog
*_log
) : cct(_cct
), log(_log
), lock("ChangesRenewThread::lock") {}
440 void *entry() override
;
444 ChangesRenewThread
*renew_thread
;
448 RGWDataChangesLog(CephContext
*_cct
, RGWRados
*_store
) : cct(_cct
), store(_store
),
449 lock("RGWDataChangesLog::lock"), modified_lock("RGWDataChangesLog::modified_lock"),
450 changes(cct
->_conf
->rgw_data_log_changes_size
) {
451 num_shards
= cct
->_conf
->rgw_data_log_num_shards
;
453 oids
= new string
[num_shards
];
455 string prefix
= cct
->_conf
->rgw_data_log_obj_prefix
;
457 if (prefix
.empty()) {
461 for (int i
= 0; i
< num_shards
; i
++) {
463 snprintf(buf
, sizeof(buf
), "%s.%d", prefix
.c_str(), i
);
467 renew_thread
= new ChangesRenewThread(cct
, this);
468 renew_thread
->create("rgw_dt_lg_renew");
471 ~RGWDataChangesLog();
473 int choose_oid(const rgw_bucket_shard
& bs
);
474 const std::string
& get_oid(int shard_id
) const { return oids
[shard_id
]; }
475 int add_entry(rgw_bucket
& bucket
, int shard_id
);
476 int get_log_shard_id(rgw_bucket
& bucket
, int shard_id
);
478 int list_entries(int shard
, const real_time
& start_time
, const real_time
& end_time
, int max_entries
,
479 list
<rgw_data_change_log_entry
>& entries
,
480 const string
& marker
,
483 int trim_entries(int shard_id
, const real_time
& start_time
, const real_time
& end_time
,
484 const string
& start_marker
, const string
& end_marker
);
485 int trim_entries(const real_time
& start_time
, const real_time
& end_time
,
486 const string
& start_marker
, const string
& end_marker
);
487 int get_info(int shard_id
, RGWDataChangesLogInfo
*info
);
488 int lock_exclusive(int shard_id
, timespan duration
, string
& zone_id
, string
& owner_id
) {
489 return store
->lock_exclusive(store
->get_zone_params().log_pool
, oids
[shard_id
], duration
, zone_id
, owner_id
);
491 int unlock(int shard_id
, string
& zone_id
, string
& owner_id
) {
492 return store
->unlock(store
->get_zone_params().log_pool
, oids
[shard_id
], zone_id
, owner_id
);
498 LogMarker() : shard(0) {}
500 int list_entries(const real_time
& start_time
, const real_time
& end_time
, int max_entries
,
501 list
<rgw_data_change_log_entry
>& entries
, LogMarker
& marker
, bool *ptruncated
);
503 void mark_modified(int shard_id
, const rgw_bucket_shard
& bs
);
504 void read_clear_modified(map
<int, set
<string
> > &modified
);