]>
Commit | Line | Data |
---|---|---|
11fdf7f2 TL |
1 | // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- |
2 | // vim: ts=8 sw=2 smarttab | |
3 | ||
7c673cae FG |
4 | #ifndef CEPH_RGW_LC_H |
5 | #define CEPH_RGW_LC_H | |
6 | ||
7 | #include <map> | |
8 | #include <string> | |
9 | #include <iostream> | |
7c673cae FG |
10 | |
11 | #include "common/debug.h" | |
12 | ||
13 | #include "include/types.h" | |
14 | #include "include/rados/librados.hpp" | |
15 | #include "common/Mutex.h" | |
16 | #include "common/Cond.h" | |
224ce89b | 17 | #include "common/iso_8601.h" |
7c673cae FG |
18 | #include "common/Thread.h" |
19 | #include "rgw_common.h" | |
20 | #include "rgw_rados.h" | |
21 | #include "rgw_multi.h" | |
22 | #include "cls/rgw/cls_rgw_types.h" | |
11fdf7f2 | 23 | #include "rgw_tag.h" |
7c673cae FG |
24 | |
25 | #include <atomic> | |
26 | ||
7c673cae FG |
27 | #define HASH_PRIME 7877 |
28 | #define MAX_ID_LEN 255 | |
29 | static string lc_oid_prefix = "lc"; | |
30 | static string lc_index_lock_name = "lc_process"; | |
31 | ||
32 | extern const char* LC_STATUS[]; | |
33 | ||
34 | typedef enum { | |
35 | lc_uninitial = 0, | |
36 | lc_processing, | |
37 | lc_failed, | |
38 | lc_complete, | |
11fdf7f2 | 39 | } LC_BUCKET_STATUS; |
7c673cae FG |
40 | |
41 | class LCExpiration | |
42 | { | |
43 | protected: | |
44 | string days; | |
224ce89b WB |
45 | //At present only current object has expiration date |
46 | string date; | |
7c673cae FG |
47 | public: |
48 | LCExpiration() {} | |
11fdf7f2 | 49 | LCExpiration(const string& _days, const string& _date) : days(_days), date(_date) {} |
7c673cae FG |
50 | |
51 | void encode(bufferlist& bl) const { | |
224ce89b | 52 | ENCODE_START(3, 2, bl); |
11fdf7f2 TL |
53 | encode(days, bl); |
54 | encode(date, bl); | |
7c673cae FG |
55 | ENCODE_FINISH(bl); |
56 | } | |
11fdf7f2 | 57 | void decode(bufferlist::const_iterator& bl) { |
224ce89b | 58 | DECODE_START_LEGACY_COMPAT_LEN(3, 2, 2, bl); |
11fdf7f2 | 59 | decode(days, bl); |
224ce89b | 60 | if (struct_v >= 3) { |
11fdf7f2 | 61 | decode(date, bl); |
224ce89b | 62 | } |
7c673cae FG |
63 | DECODE_FINISH(bl); |
64 | } | |
65 | void dump(Formatter *f) const; | |
66 | // static void generate_test_instances(list<ACLOwner*>& o); | |
67 | void set_days(const string& _days) { days = _days; } | |
224ce89b | 68 | string get_days_str() const { |
31f18b77 FG |
69 | return days; |
70 | } | |
224ce89b WB |
71 | int get_days() const {return atoi(days.c_str()); } |
72 | bool has_days() const { | |
73 | return !days.empty(); | |
74 | } | |
75 | void set_date(const string& _date) { date = _date; } | |
76 | string get_date() const { | |
77 | return date; | |
78 | } | |
79 | bool has_date() const { | |
80 | return !date.empty(); | |
81 | } | |
82 | bool empty() const { | |
83 | return days.empty() && date.empty(); | |
84 | } | |
85 | bool valid() const { | |
86 | if (!days.empty() && !date.empty()) { | |
87 | return false; | |
88 | } else if (!days.empty() && get_days() <= 0) { | |
89 | return false; | |
90 | } | |
91 | //We've checked date in xml parsing | |
92 | return true; | |
7c673cae FG |
93 | } |
94 | }; | |
95 | WRITE_CLASS_ENCODER(LCExpiration) | |
96 | ||
11fdf7f2 TL |
97 | class LCTransition |
98 | { | |
99 | protected: | |
100 | string days; | |
101 | string date; | |
102 | string storage_class; | |
103 | ||
104 | public: | |
105 | int get_days() const { | |
106 | return atoi(days.c_str()); | |
107 | } | |
108 | ||
109 | string get_date() const { | |
110 | return date; | |
111 | } | |
112 | ||
113 | string get_storage_class() const { | |
114 | return storage_class; | |
115 | } | |
116 | ||
117 | bool has_days() const { | |
118 | return !days.empty(); | |
119 | } | |
120 | ||
121 | bool has_date() const { | |
122 | return !date.empty(); | |
123 | } | |
124 | ||
125 | bool empty() const { | |
126 | return days.empty() && date.empty(); | |
127 | } | |
128 | ||
129 | bool valid() const { | |
130 | if (!days.empty() && !date.empty()) { | |
131 | return false; | |
132 | } else if (!days.empty() && get_days() <=0) { | |
133 | return false; | |
134 | } | |
135 | //We've checked date in xml parsing | |
136 | return true; | |
137 | } | |
138 | ||
139 | void encode(bufferlist& bl) const { | |
140 | ENCODE_START(1, 1, bl); | |
141 | encode(days, bl); | |
142 | encode(date, bl); | |
143 | encode(storage_class, bl); | |
144 | ENCODE_FINISH(bl); | |
145 | } | |
146 | ||
147 | void decode(bufferlist::const_iterator& bl) { | |
148 | DECODE_START(1, bl); | |
149 | decode(days, bl); | |
150 | decode(date, bl); | |
151 | decode(storage_class, bl); | |
152 | DECODE_FINISH(bl); | |
153 | } | |
154 | void dump(Formatter *f) const; | |
155 | }; | |
156 | WRITE_CLASS_ENCODER(LCTransition) | |
157 | ||
181888fb FG |
158 | class LCFilter |
159 | { | |
160 | protected: | |
161 | std::string prefix; | |
11fdf7f2 TL |
162 | RGWObjTags obj_tags; |
163 | ||
181888fb | 164 | public: |
11fdf7f2 TL |
165 | |
166 | const std::string& get_prefix() const { | |
181888fb FG |
167 | return prefix; |
168 | } | |
169 | ||
11fdf7f2 TL |
170 | const RGWObjTags& get_tags() const { |
171 | return obj_tags; | |
181888fb FG |
172 | } |
173 | ||
11fdf7f2 TL |
174 | bool empty() const { |
175 | return !(has_prefix() || has_tags()); | |
181888fb FG |
176 | } |
177 | ||
11fdf7f2 TL |
178 | // Determine if we need AND tag when creating xml |
179 | bool has_multi_condition() const { | |
180 | if (obj_tags.count() > 1) | |
181 | return true; | |
182 | else if (has_prefix() && has_tags()) | |
183 | return true; | |
184 | ||
185 | return false; | |
181888fb FG |
186 | } |
187 | ||
188 | bool has_prefix() const { | |
189 | return !prefix.empty(); | |
190 | } | |
191 | ||
11fdf7f2 TL |
192 | bool has_tags() const { |
193 | return !obj_tags.empty(); | |
194 | } | |
195 | ||
181888fb | 196 | void encode(bufferlist& bl) const { |
11fdf7f2 TL |
197 | ENCODE_START(2, 1, bl); |
198 | encode(prefix, bl); | |
199 | encode(obj_tags, bl); | |
181888fb FG |
200 | ENCODE_FINISH(bl); |
201 | } | |
11fdf7f2 TL |
202 | void decode(bufferlist::const_iterator& bl) { |
203 | DECODE_START(2, bl); | |
204 | decode(prefix, bl); | |
205 | if (struct_v >= 2) { | |
206 | decode(obj_tags, bl); | |
207 | } | |
181888fb FG |
208 | DECODE_FINISH(bl); |
209 | } | |
11fdf7f2 | 210 | void dump(Formatter *f) const; |
181888fb | 211 | }; |
11fdf7f2 | 212 | WRITE_CLASS_ENCODER(LCFilter) |
181888fb FG |
213 | |
214 | ||
215 | ||
7c673cae FG |
216 | class LCRule |
217 | { | |
218 | protected: | |
219 | string id; | |
220 | string prefix; | |
221 | string status; | |
222 | LCExpiration expiration; | |
223 | LCExpiration noncur_expiration; | |
224 | LCExpiration mp_expiration; | |
181888fb | 225 | LCFilter filter; |
11fdf7f2 TL |
226 | map<string, LCTransition> transitions; |
227 | map<string, LCTransition> noncur_transitions; | |
224ce89b | 228 | bool dm_expiration = false; |
7c673cae FG |
229 | |
230 | public: | |
231 | ||
232 | LCRule(){}; | |
233 | ~LCRule(){}; | |
234 | ||
11fdf7f2 TL |
235 | const string& get_id() const { |
236 | return id; | |
7c673cae FG |
237 | } |
238 | ||
11fdf7f2 | 239 | const string& get_status() const { |
7c673cae FG |
240 | return status; |
241 | } | |
181888fb | 242 | |
11fdf7f2 TL |
243 | bool is_enabled() const { |
244 | return status == "Enabled"; | |
245 | } | |
246 | ||
247 | void set_enabled(bool flag) { | |
248 | status = (flag ? "Enabled" : "Disabled"); | |
249 | } | |
250 | ||
251 | const string& get_prefix() const { | |
7c673cae FG |
252 | return prefix; |
253 | } | |
254 | ||
11fdf7f2 | 255 | const LCFilter& get_filter() const { |
181888fb FG |
256 | return filter; |
257 | } | |
258 | ||
11fdf7f2 | 259 | const LCExpiration& get_expiration() const { |
7c673cae FG |
260 | return expiration; |
261 | } | |
262 | ||
11fdf7f2 | 263 | const LCExpiration& get_noncur_expiration() const { |
7c673cae FG |
264 | return noncur_expiration; |
265 | } | |
266 | ||
11fdf7f2 | 267 | const LCExpiration& get_mp_expiration() const { |
7c673cae FG |
268 | return mp_expiration; |
269 | } | |
270 | ||
11fdf7f2 | 271 | bool get_dm_expiration() const { |
31f18b77 FG |
272 | return dm_expiration; |
273 | } | |
274 | ||
11fdf7f2 TL |
275 | const map<string, LCTransition>& get_transitions() const { |
276 | return transitions; | |
277 | } | |
278 | ||
279 | const map<string, LCTransition>& get_noncur_transitions() const { | |
280 | return noncur_transitions; | |
7c673cae FG |
281 | } |
282 | ||
11fdf7f2 TL |
283 | void set_id(const string& _id) { |
284 | id = _id; | |
285 | } | |
286 | ||
287 | void set_prefix(const string& _prefix) { | |
288 | prefix = _prefix; | |
7c673cae FG |
289 | } |
290 | ||
11fdf7f2 TL |
291 | void set_status(const string& _status) { |
292 | status = _status; | |
7c673cae FG |
293 | } |
294 | ||
11fdf7f2 TL |
295 | void set_expiration(const LCExpiration& _expiration) { |
296 | expiration = _expiration; | |
7c673cae FG |
297 | } |
298 | ||
11fdf7f2 TL |
299 | void set_noncur_expiration(const LCExpiration& _noncur_expiration) { |
300 | noncur_expiration = _noncur_expiration; | |
7c673cae FG |
301 | } |
302 | ||
11fdf7f2 TL |
303 | void set_mp_expiration(const LCExpiration& _mp_expiration) { |
304 | mp_expiration = _mp_expiration; | |
7c673cae FG |
305 | } |
306 | ||
31f18b77 FG |
307 | void set_dm_expiration(bool _dm_expiration) { |
308 | dm_expiration = _dm_expiration; | |
309 | } | |
310 | ||
11fdf7f2 TL |
311 | bool add_transition(const LCTransition& _transition) { |
312 | auto ret = transitions.emplace(_transition.get_storage_class(), _transition); | |
313 | return ret.second; | |
314 | } | |
315 | ||
316 | bool add_noncur_transition(const LCTransition& _noncur_transition) { | |
317 | auto ret = noncur_transitions.emplace(_noncur_transition.get_storage_class(), _noncur_transition); | |
318 | return ret.second; | |
319 | } | |
320 | ||
321 | bool valid() const; | |
7c673cae FG |
322 | |
323 | void encode(bufferlist& bl) const { | |
11fdf7f2 TL |
324 | ENCODE_START(6, 1, bl); |
325 | encode(id, bl); | |
326 | encode(prefix, bl); | |
327 | encode(status, bl); | |
328 | encode(expiration, bl); | |
329 | encode(noncur_expiration, bl); | |
330 | encode(mp_expiration, bl); | |
331 | encode(dm_expiration, bl); | |
332 | encode(filter, bl); | |
333 | encode(transitions, bl); | |
334 | encode(noncur_transitions, bl); | |
7c673cae FG |
335 | ENCODE_FINISH(bl); |
336 | } | |
11fdf7f2 TL |
337 | void decode(bufferlist::const_iterator& bl) { |
338 | DECODE_START_LEGACY_COMPAT_LEN(6, 1, 1, bl); | |
339 | decode(id, bl); | |
340 | decode(prefix, bl); | |
341 | decode(status, bl); | |
342 | decode(expiration, bl); | |
7c673cae | 343 | if (struct_v >=2) { |
11fdf7f2 | 344 | decode(noncur_expiration, bl); |
7c673cae FG |
345 | } |
346 | if (struct_v >= 3) { | |
11fdf7f2 | 347 | decode(mp_expiration, bl); |
7c673cae | 348 | } |
31f18b77 | 349 | if (struct_v >= 4) { |
11fdf7f2 | 350 | decode(dm_expiration, bl); |
31f18b77 | 351 | } |
181888fb | 352 | if (struct_v >= 5) { |
11fdf7f2 TL |
353 | decode(filter, bl); |
354 | } | |
355 | if (struct_v >= 6) { | |
356 | decode(transitions, bl); | |
357 | decode(noncur_transitions, bl); | |
181888fb | 358 | } |
7c673cae FG |
359 | DECODE_FINISH(bl); |
360 | } | |
11fdf7f2 | 361 | void dump(Formatter *f) const; |
7c673cae | 362 | |
11fdf7f2 | 363 | void init_simple_days_rule(std::string_view _id, std::string_view _prefix, int num_days); |
7c673cae FG |
364 | }; |
365 | WRITE_CLASS_ENCODER(LCRule) | |
366 | ||
11fdf7f2 TL |
367 | struct transition_action |
368 | { | |
369 | int days; | |
370 | boost::optional<ceph::real_time> date; | |
371 | string storage_class; | |
372 | transition_action() : days(0) {} | |
373 | }; | |
374 | ||
7c673cae FG |
375 | struct lc_op |
376 | { | |
11fdf7f2 TL |
377 | bool status{false}; |
378 | bool dm_expiration{false}; | |
379 | int expiration{0}; | |
380 | int noncur_expiration{0}; | |
381 | int mp_expiration{0}; | |
224ce89b | 382 | boost::optional<ceph::real_time> expiration_date; |
11fdf7f2 TL |
383 | boost::optional<RGWObjTags> obj_tags; |
384 | map<string, transition_action> transitions; | |
385 | map<string, transition_action> noncur_transitions; | |
7c673cae | 386 | |
11fdf7f2 | 387 | void dump(Formatter *f) const; |
7c673cae FG |
388 | }; |
389 | ||
390 | class RGWLifecycleConfiguration | |
391 | { | |
392 | protected: | |
393 | CephContext *cct; | |
394 | map<string, lc_op> prefix_map; | |
395 | multimap<string, LCRule> rule_map; | |
11fdf7f2 | 396 | bool _add_rule(const LCRule& rule); |
224ce89b | 397 | bool has_same_action(const lc_op& first, const lc_op& second); |
7c673cae | 398 | public: |
11fdf7f2 | 399 | explicit RGWLifecycleConfiguration(CephContext *_cct) : cct(_cct) {} |
7c673cae FG |
400 | RGWLifecycleConfiguration() : cct(NULL) {} |
401 | ||
402 | void set_ctx(CephContext *ctx) { | |
403 | cct = ctx; | |
404 | } | |
405 | ||
406 | virtual ~RGWLifecycleConfiguration() {} | |
407 | ||
408 | // int get_perm(string& id, int perm_mask); | |
409 | // int get_group_perm(ACLGroupTypeEnum group, int perm_mask); | |
410 | void encode(bufferlist& bl) const { | |
411 | ENCODE_START(1, 1, bl); | |
11fdf7f2 | 412 | encode(rule_map, bl); |
7c673cae FG |
413 | ENCODE_FINISH(bl); |
414 | } | |
11fdf7f2 | 415 | void decode(bufferlist::const_iterator& bl) { |
7c673cae | 416 | DECODE_START_LEGACY_COMPAT_LEN(1, 1, 1, bl); |
11fdf7f2 | 417 | decode(rule_map, bl); |
7c673cae FG |
418 | multimap<string, LCRule>::iterator iter; |
419 | for (iter = rule_map.begin(); iter != rule_map.end(); ++iter) { | |
420 | LCRule& rule = iter->second; | |
11fdf7f2 | 421 | _add_rule(rule); |
7c673cae FG |
422 | } |
423 | DECODE_FINISH(bl); | |
424 | } | |
425 | void dump(Formatter *f) const; | |
11fdf7f2 | 426 | static void generate_test_instances(list<RGWLifecycleConfiguration*>& o); |
7c673cae | 427 | |
11fdf7f2 | 428 | void add_rule(const LCRule& rule); |
7c673cae | 429 | |
11fdf7f2 | 430 | int check_and_add_rule(const LCRule& rule); |
7c673cae | 431 | |
224ce89b | 432 | bool valid(); |
7c673cae FG |
433 | |
434 | multimap<string, LCRule>& get_rule_map() { return rule_map; } | |
435 | map<string, lc_op>& get_prefix_map() { return prefix_map; } | |
436 | /* | |
437 | void create_default(string id, string name) { | |
438 | ACLGrant grant; | |
439 | grant.set_canon(id, name, RGW_PERM_FULL_CONTROL); | |
440 | add_grant(&grant); | |
441 | } | |
442 | */ | |
443 | }; | |
444 | WRITE_CLASS_ENCODER(RGWLifecycleConfiguration) | |
445 | ||
11fdf7f2 | 446 | class RGWLC : public DoutPrefixProvider { |
7c673cae FG |
447 | CephContext *cct; |
448 | RGWRados *store; | |
224ce89b WB |
449 | int max_objs{0}; |
450 | string *obj_names{nullptr}; | |
7c673cae FG |
451 | std::atomic<bool> down_flag = { false }; |
452 | string cookie; | |
453 | ||
454 | class LCWorker : public Thread { | |
11fdf7f2 | 455 | const DoutPrefixProvider *dpp; |
7c673cae FG |
456 | CephContext *cct; |
457 | RGWLC *lc; | |
458 | Mutex lock; | |
459 | Cond cond; | |
460 | ||
461 | public: | |
11fdf7f2 | 462 | LCWorker(const DoutPrefixProvider* _dpp, CephContext *_cct, RGWLC *_lc) : dpp(_dpp), cct(_cct), lc(_lc), lock("LCWorker") {} |
7c673cae FG |
463 | void *entry() override; |
464 | void stop(); | |
465 | bool should_work(utime_t& now); | |
466 | int schedule_next_start_time(utime_t& start, utime_t& now); | |
467 | }; | |
468 | ||
469 | public: | |
470 | LCWorker *worker; | |
471 | RGWLC() : cct(NULL), store(NULL), worker(NULL) {} | |
472 | ~RGWLC() { | |
473 | stop_processor(); | |
474 | finalize(); | |
475 | } | |
476 | ||
477 | void initialize(CephContext *_cct, RGWRados *_store); | |
478 | void finalize(); | |
479 | ||
480 | int process(); | |
481 | int process(int index, int max_secs); | |
482 | bool if_already_run_today(time_t& start_date); | |
483 | int list_lc_progress(const string& marker, uint32_t max_entries, map<string, int> *progress_map); | |
484 | int bucket_lc_prepare(int index); | |
485 | int bucket_lc_process(string& shard_id); | |
486 | int bucket_lc_post(int index, int max_lock_sec, pair<string, int >& entry, int& result); | |
487 | bool going_down(); | |
488 | void start_processor(); | |
489 | void stop_processor(); | |
11fdf7f2 TL |
490 | int set_bucket_config(RGWBucketInfo& bucket_info, |
491 | const map<string, bufferlist>& bucket_attrs, | |
492 | RGWLifecycleConfiguration *config); | |
493 | int remove_bucket_config(RGWBucketInfo& bucket_info, | |
494 | const map<string, bufferlist>& bucket_attrs); | |
495 | ||
496 | CephContext *get_cct() const override { return store->ctx(); } | |
497 | unsigned get_subsys() const; | |
498 | std::ostream& gen_prefix(std::ostream& out) const; | |
7c673cae FG |
499 | |
500 | private: | |
11fdf7f2 | 501 | |
7c673cae FG |
502 | int handle_multipart_expiration(RGWRados::Bucket *target, const map<string, lc_op>& prefix_map); |
503 | }; | |
504 | ||
11fdf7f2 TL |
505 | namespace rgw::lc { |
506 | ||
507 | int fix_lc_shard_entry(RGWRados *store, const RGWBucketInfo& bucket_info, | |
508 | const map<std::string,bufferlist>& battrs); | |
7c673cae | 509 | |
11fdf7f2 | 510 | } // namespace rgw::lc |
7c673cae FG |
511 | |
512 | #endif |