1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab ft=cpp
11 #include "common/debug.h"
13 #include "include/types.h"
14 #include "include/rados/librados.hpp"
15 #include "common/ceph_mutex.h"
16 #include "common/Cond.h"
17 #include "common/iso_8601.h"
18 #include "common/Thread.h"
19 #include "rgw_common.h"
20 #include "rgw_rados.h"
21 #include "cls/rgw/cls_rgw_types.h"
28 #define HASH_PRIME 7877
29 #define MAX_ID_LEN 255
30 static string lc_oid_prefix
= "lc";
31 static string lc_index_lock_name
= "lc_process";
33 extern const char* LC_STATUS
[];
46 //At present only current object has expiration date
50 LCExpiration(const string
& _days
, const string
& _date
) : days(_days
), date(_date
) {}
52 void encode(bufferlist
& bl
) const {
53 ENCODE_START(3, 2, bl
);
58 void decode(bufferlist::const_iterator
& bl
) {
59 DECODE_START_LEGACY_COMPAT_LEN(3, 2, 2, bl
);
66 void dump(Formatter
*f
) const;
67 // static void generate_test_instances(list<ACLOwner*>& o);
68 void set_days(const string
& _days
) { days
= _days
; }
69 string
get_days_str() const {
72 int get_days() const {return atoi(days
.c_str()); }
73 bool has_days() const {
76 void set_date(const string
& _date
) { date
= _date
; }
77 string
get_date() const {
80 bool has_date() const {
84 return days
.empty() && date
.empty();
87 if (!days
.empty() && !date
.empty()) {
89 } else if (!days
.empty() && get_days() <= 0) {
92 //We've checked date in xml parsing
96 WRITE_CLASS_ENCODER(LCExpiration
)
103 string storage_class
;
106 int get_days() const {
107 return atoi(days
.c_str());
110 string
get_date() const {
114 string
get_storage_class() const {
115 return storage_class
;
118 bool has_days() const {
119 return !days
.empty();
122 bool has_date() const {
123 return !date
.empty();
127 return days
.empty() && date
.empty();
131 if (!days
.empty() && !date
.empty()) {
133 } else if (!days
.empty() && get_days() < 0) {
136 //We've checked date in xml parsing
140 void encode(bufferlist
& bl
) const {
141 ENCODE_START(1, 1, bl
);
144 encode(storage_class
, bl
);
148 void decode(bufferlist::const_iterator
& bl
) {
152 decode(storage_class
, bl
);
155 void dump(Formatter
*f
) const;
157 WRITE_CLASS_ENCODER(LCTransition
)
167 const std::string
& get_prefix() const {
171 const RGWObjTags
& get_tags() const {
176 return !(has_prefix() || has_tags());
179 // Determine if we need AND tag when creating xml
180 bool has_multi_condition() const {
181 if (obj_tags
.count() > 1)
183 else if (has_prefix() && has_tags())
189 bool has_prefix() const {
190 return !prefix
.empty();
193 bool has_tags() const {
194 return !obj_tags
.empty();
197 void encode(bufferlist
& bl
) const {
198 ENCODE_START(2, 1, bl
);
200 encode(obj_tags
, bl
);
203 void decode(bufferlist::const_iterator
& bl
) {
207 decode(obj_tags
, bl
);
211 void dump(Formatter
*f
) const;
213 WRITE_CLASS_ENCODER(LCFilter
)
221 LCExpiration expiration
;
222 LCExpiration noncur_expiration
;
223 LCExpiration mp_expiration
;
225 map
<string
, LCTransition
> transitions
;
226 map
<string
, LCTransition
> noncur_transitions
;
227 bool dm_expiration
= false;
234 const string
& get_id() const {
238 const string
& get_status() const {
242 bool is_enabled() const {
243 return status
== "Enabled";
246 void set_enabled(bool flag
) {
247 status
= (flag
? "Enabled" : "Disabled");
250 const string
& get_prefix() const {
254 const LCFilter
& get_filter() const {
258 const LCExpiration
& get_expiration() const {
262 const LCExpiration
& get_noncur_expiration() const {
263 return noncur_expiration
;
266 const LCExpiration
& get_mp_expiration() const {
267 return mp_expiration
;
270 bool get_dm_expiration() const {
271 return dm_expiration
;
274 const map
<string
, LCTransition
>& get_transitions() const {
278 const map
<string
, LCTransition
>& get_noncur_transitions() const {
279 return noncur_transitions
;
282 void set_id(const string
& _id
) {
286 void set_prefix(const string
& _prefix
) {
290 void set_status(const string
& _status
) {
294 void set_expiration(const LCExpiration
& _expiration
) {
295 expiration
= _expiration
;
298 void set_noncur_expiration(const LCExpiration
& _noncur_expiration
) {
299 noncur_expiration
= _noncur_expiration
;
302 void set_mp_expiration(const LCExpiration
& _mp_expiration
) {
303 mp_expiration
= _mp_expiration
;
306 void set_dm_expiration(bool _dm_expiration
) {
307 dm_expiration
= _dm_expiration
;
310 bool add_transition(const LCTransition
& _transition
) {
311 auto ret
= transitions
.emplace(_transition
.get_storage_class(), _transition
);
315 bool add_noncur_transition(const LCTransition
& _noncur_transition
) {
316 auto ret
= noncur_transitions
.emplace(_noncur_transition
.get_storage_class(), _noncur_transition
);
322 void encode(bufferlist
& bl
) const {
323 ENCODE_START(6, 1, bl
);
327 encode(expiration
, bl
);
328 encode(noncur_expiration
, bl
);
329 encode(mp_expiration
, bl
);
330 encode(dm_expiration
, bl
);
332 encode(transitions
, bl
);
333 encode(noncur_transitions
, bl
);
336 void decode(bufferlist::const_iterator
& bl
) {
337 DECODE_START_LEGACY_COMPAT_LEN(6, 1, 1, bl
);
341 decode(expiration
, bl
);
343 decode(noncur_expiration
, bl
);
346 decode(mp_expiration
, bl
);
349 decode(dm_expiration
, bl
);
355 decode(transitions
, bl
);
356 decode(noncur_transitions
, bl
);
360 void dump(Formatter
*f
) const;
362 void init_simple_days_rule(std::string_view _id
, std::string_view _prefix
, int num_days
);
364 WRITE_CLASS_ENCODER(LCRule
)
366 struct transition_action
369 boost::optional
<ceph::real_time
> date
;
370 string storage_class
;
371 transition_action() : days(0) {}
374 /* XXX why not LCRule? */
379 bool dm_expiration
{false};
381 int noncur_expiration
{0};
382 int mp_expiration
{0};
383 boost::optional
<ceph::real_time
> expiration_date
;
384 boost::optional
<RGWObjTags
> obj_tags
;
385 map
<string
, transition_action
> transitions
;
386 map
<string
, transition_action
> noncur_transitions
;
391 lc_op(const std::string id
) : id(id
)
394 void dump(Formatter
*f
) const;
397 class RGWLifecycleConfiguration
401 multimap
<string
, lc_op
> prefix_map
;
402 multimap
<string
, LCRule
> rule_map
;
403 bool _add_rule(const LCRule
& rule
);
404 bool has_same_action(const lc_op
& first
, const lc_op
& second
);
406 explicit RGWLifecycleConfiguration(CephContext
*_cct
) : cct(_cct
) {}
407 RGWLifecycleConfiguration() : cct(NULL
) {}
409 void set_ctx(CephContext
*ctx
) {
413 virtual ~RGWLifecycleConfiguration() {}
415 // int get_perm(string& id, int perm_mask);
416 // int get_group_perm(ACLGroupTypeEnum group, int perm_mask);
417 void encode(bufferlist
& bl
) const {
418 ENCODE_START(1, 1, bl
);
419 encode(rule_map
, bl
);
422 void decode(bufferlist::const_iterator
& bl
) {
423 DECODE_START_LEGACY_COMPAT_LEN(1, 1, 1, bl
);
424 decode(rule_map
, bl
);
425 multimap
<string
, LCRule
>::iterator iter
;
426 for (iter
= rule_map
.begin(); iter
!= rule_map
.end(); ++iter
) {
427 LCRule
& rule
= iter
->second
;
432 void dump(Formatter
*f
) const;
433 static void generate_test_instances(list
<RGWLifecycleConfiguration
*>& o
);
435 void add_rule(const LCRule
& rule
);
437 int check_and_add_rule(const LCRule
& rule
);
441 multimap
<string
, LCRule
>& get_rule_map() { return rule_map
; }
442 multimap
<string
, lc_op
>& get_prefix_map() { return prefix_map
; }
444 void create_default(string id, string name) {
446 grant.set_canon(id, name, RGW_PERM_FULL_CONTROL);
451 WRITE_CLASS_ENCODER(RGWLifecycleConfiguration
)
453 class RGWLC
: public DoutPrefixProvider
{
455 rgw::sal::RGWRadosStore
*store
;
457 string
*obj_names
{nullptr};
458 std::atomic
<bool> down_flag
= { false };
465 class LCWorker
: public Thread
467 const DoutPrefixProvider
*dpp
;
472 std::condition_variable cond
;
473 WorkPool
* workpool
{nullptr};
477 using lock_guard
= std::lock_guard
<std::mutex
>;
478 using unique_lock
= std::unique_lock
<std::mutex
>;
480 LCWorker(const DoutPrefixProvider
* dpp
, CephContext
*_cct
, RGWLC
*_lc
,
482 RGWLC
* get_lc() { return lc
; }
483 void *entry() override
;
485 bool should_work(utime_t
& now
);
486 int schedule_next_start_time(utime_t
& start
, utime_t
& now
);
489 friend class RGWRados
;
494 friend class RGWRados
;
496 std::vector
<std::unique_ptr
<RGWLC::LCWorker
>> workers
;
498 RGWLC() : cct(nullptr), store(nullptr) {}
501 void initialize(CephContext
*_cct
, rgw::sal::RGWRadosStore
*_store
);
504 int process(LCWorker
* worker
, bool once
);
505 int process(int index
, int max_secs
, LCWorker
* worker
, bool once
);
506 bool if_already_run_today(time_t start_date
);
507 bool expired_session(time_t started
);
508 time_t thread_stop_at();
509 int list_lc_progress(string
& marker
, uint32_t max_entries
,
510 vector
<cls_rgw_lc_entry
>&, int& index
);
511 int bucket_lc_prepare(int index
, LCWorker
* worker
);
512 int bucket_lc_process(string
& shard_id
, LCWorker
* worker
, time_t stop_at
,
514 int bucket_lc_post(int index
, int max_lock_sec
,
515 cls_rgw_lc_entry
& entry
, int& result
, LCWorker
* worker
);
517 void start_processor();
518 void stop_processor();
519 int set_bucket_config(RGWBucketInfo
& bucket_info
,
520 const map
<string
, bufferlist
>& bucket_attrs
,
521 RGWLifecycleConfiguration
*config
);
522 int remove_bucket_config(RGWBucketInfo
& bucket_info
,
523 const map
<string
, bufferlist
>& bucket_attrs
);
525 CephContext
*get_cct() const override
{ return store
->ctx(); }
526 unsigned get_subsys() const;
527 std::ostream
& gen_prefix(std::ostream
& out
) const;
531 int handle_multipart_expiration(RGWRados::Bucket
*target
,
532 const multimap
<string
, lc_op
>& prefix_map
,
533 LCWorker
* worker
, time_t stop_at
, bool once
);
538 int fix_lc_shard_entry(rgw::sal::RGWRadosStore
*store
, const RGWBucketInfo
& bucket_info
,
539 const map
<std::string
,bufferlist
>& battrs
);
541 std::string
s3_expiration_header(
542 DoutPrefixProvider
* dpp
,
543 const rgw_obj_key
& obj_key
,
544 const RGWObjTags
& obj_tagset
,
545 const ceph::real_time
& mtime
,
546 const std::map
<std::string
, buffer::list
>& bucket_attrs
);
548 bool s3_multipart_abort_header(
549 DoutPrefixProvider
* dpp
,
550 const rgw_obj_key
& obj_key
,
551 const ceph::real_time
& mtime
,
552 const std::map
<std::string
, buffer::list
>& bucket_attrs
,
553 ceph::real_time
& abort_date
,
554 std::string
& rule_id
);
556 } // namespace rgw::lc