]>
Commit | Line | Data |
---|---|---|
11fdf7f2 | 1 | // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- |
9f95a23c | 2 | // vim: ts=8 sw=2 smarttab ft=cpp |
11fdf7f2 | 3 | |
7c673cae FG |
4 | #ifndef CEPH_RGW_LC_H |
5 | #define CEPH_RGW_LC_H | |
6 | ||
7 | #include <map> | |
8 | #include <string> | |
9 | #include <iostream> | |
7c673cae FG |
10 | |
11 | #include "common/debug.h" | |
12 | ||
13 | #include "include/types.h" | |
14 | #include "include/rados/librados.hpp" | |
9f95a23c | 15 | #include "common/ceph_mutex.h" |
7c673cae | 16 | #include "common/Cond.h" |
224ce89b | 17 | #include "common/iso_8601.h" |
7c673cae FG |
18 | #include "common/Thread.h" |
19 | #include "rgw_common.h" | |
20 | #include "rgw_rados.h" | |
7c673cae | 21 | #include "cls/rgw/cls_rgw_types.h" |
11fdf7f2 | 22 | #include "rgw_tag.h" |
9f95a23c | 23 | #include "rgw_sal.h" |
7c673cae FG |
24 | |
25 | #include <atomic> | |
9f95a23c | 26 | #include <tuple> |
7c673cae | 27 | |
7c673cae FG |
28 | #define HASH_PRIME 7877 |
29 | #define MAX_ID_LEN 255 | |
30 | static string lc_oid_prefix = "lc"; | |
31 | static string lc_index_lock_name = "lc_process"; | |
32 | ||
33 | extern const char* LC_STATUS[]; | |
34 | ||
35 | typedef enum { | |
36 | lc_uninitial = 0, | |
37 | lc_processing, | |
38 | lc_failed, | |
39 | lc_complete, | |
11fdf7f2 | 40 | } LC_BUCKET_STATUS; |
7c673cae FG |
41 | |
42 | class LCExpiration | |
43 | { | |
44 | protected: | |
45 | string days; | |
224ce89b WB |
46 | //At present only current object has expiration date |
47 | string date; | |
7c673cae FG |
48 | public: |
49 | LCExpiration() {} | |
11fdf7f2 | 50 | LCExpiration(const string& _days, const string& _date) : days(_days), date(_date) {} |
7c673cae FG |
51 | |
52 | void encode(bufferlist& bl) const { | |
224ce89b | 53 | ENCODE_START(3, 2, bl); |
11fdf7f2 TL |
54 | encode(days, bl); |
55 | encode(date, bl); | |
7c673cae FG |
56 | ENCODE_FINISH(bl); |
57 | } | |
11fdf7f2 | 58 | void decode(bufferlist::const_iterator& bl) { |
224ce89b | 59 | DECODE_START_LEGACY_COMPAT_LEN(3, 2, 2, bl); |
11fdf7f2 | 60 | decode(days, bl); |
224ce89b | 61 | if (struct_v >= 3) { |
11fdf7f2 | 62 | decode(date, bl); |
224ce89b | 63 | } |
7c673cae FG |
64 | DECODE_FINISH(bl); |
65 | } | |
66 | void dump(Formatter *f) const; | |
67 | // static void generate_test_instances(list<ACLOwner*>& o); | |
68 | void set_days(const string& _days) { days = _days; } | |
224ce89b | 69 | string get_days_str() const { |
31f18b77 FG |
70 | return days; |
71 | } | |
224ce89b WB |
72 | int get_days() const {return atoi(days.c_str()); } |
73 | bool has_days() const { | |
74 | return !days.empty(); | |
75 | } | |
76 | void set_date(const string& _date) { date = _date; } | |
77 | string get_date() const { | |
78 | return date; | |
79 | } | |
80 | bool has_date() const { | |
81 | return !date.empty(); | |
82 | } | |
83 | bool empty() const { | |
84 | return days.empty() && date.empty(); | |
85 | } | |
86 | bool valid() const { | |
87 | if (!days.empty() && !date.empty()) { | |
88 | return false; | |
9f95a23c | 89 | } else if (!days.empty() && get_days() <= 0) { |
224ce89b WB |
90 | return false; |
91 | } | |
92 | //We've checked date in xml parsing | |
93 | return true; | |
7c673cae FG |
94 | } |
95 | }; | |
96 | WRITE_CLASS_ENCODER(LCExpiration) | |
97 | ||
11fdf7f2 TL |
98 | class LCTransition |
99 | { | |
100 | protected: | |
101 | string days; | |
102 | string date; | |
103 | string storage_class; | |
104 | ||
105 | public: | |
106 | int get_days() const { | |
107 | return atoi(days.c_str()); | |
108 | } | |
109 | ||
110 | string get_date() const { | |
111 | return date; | |
112 | } | |
113 | ||
114 | string get_storage_class() const { | |
115 | return storage_class; | |
116 | } | |
117 | ||
118 | bool has_days() const { | |
119 | return !days.empty(); | |
120 | } | |
121 | ||
122 | bool has_date() const { | |
123 | return !date.empty(); | |
124 | } | |
125 | ||
126 | bool empty() const { | |
127 | return days.empty() && date.empty(); | |
128 | } | |
129 | ||
130 | bool valid() const { | |
131 | if (!days.empty() && !date.empty()) { | |
132 | return false; | |
9f95a23c | 133 | } else if (!days.empty() && get_days() < 0) { |
11fdf7f2 TL |
134 | return false; |
135 | } | |
136 | //We've checked date in xml parsing | |
137 | return true; | |
138 | } | |
139 | ||
140 | void encode(bufferlist& bl) const { | |
141 | ENCODE_START(1, 1, bl); | |
142 | encode(days, bl); | |
143 | encode(date, bl); | |
144 | encode(storage_class, bl); | |
145 | ENCODE_FINISH(bl); | |
146 | } | |
147 | ||
148 | void decode(bufferlist::const_iterator& bl) { | |
149 | DECODE_START(1, bl); | |
150 | decode(days, bl); | |
151 | decode(date, bl); | |
152 | decode(storage_class, bl); | |
153 | DECODE_FINISH(bl); | |
154 | } | |
155 | void dump(Formatter *f) const; | |
156 | }; | |
157 | WRITE_CLASS_ENCODER(LCTransition) | |
158 | ||
181888fb FG |
159 | class LCFilter |
160 | { | |
161 | protected: | |
162 | std::string prefix; | |
11fdf7f2 TL |
163 | RGWObjTags obj_tags; |
164 | ||
181888fb | 165 | public: |
11fdf7f2 TL |
166 | |
167 | const std::string& get_prefix() const { | |
181888fb FG |
168 | return prefix; |
169 | } | |
170 | ||
11fdf7f2 TL |
171 | const RGWObjTags& get_tags() const { |
172 | return obj_tags; | |
181888fb FG |
173 | } |
174 | ||
11fdf7f2 TL |
175 | bool empty() const { |
176 | return !(has_prefix() || has_tags()); | |
181888fb FG |
177 | } |
178 | ||
11fdf7f2 TL |
179 | // Determine if we need AND tag when creating xml |
180 | bool has_multi_condition() const { | |
181 | if (obj_tags.count() > 1) | |
182 | return true; | |
183 | else if (has_prefix() && has_tags()) | |
184 | return true; | |
185 | ||
186 | return false; | |
181888fb FG |
187 | } |
188 | ||
189 | bool has_prefix() const { | |
190 | return !prefix.empty(); | |
191 | } | |
192 | ||
11fdf7f2 TL |
193 | bool has_tags() const { |
194 | return !obj_tags.empty(); | |
195 | } | |
196 | ||
181888fb | 197 | void encode(bufferlist& bl) const { |
11fdf7f2 TL |
198 | ENCODE_START(2, 1, bl); |
199 | encode(prefix, bl); | |
200 | encode(obj_tags, bl); | |
181888fb FG |
201 | ENCODE_FINISH(bl); |
202 | } | |
11fdf7f2 TL |
203 | void decode(bufferlist::const_iterator& bl) { |
204 | DECODE_START(2, bl); | |
205 | decode(prefix, bl); | |
206 | if (struct_v >= 2) { | |
207 | decode(obj_tags, bl); | |
208 | } | |
181888fb FG |
209 | DECODE_FINISH(bl); |
210 | } | |
11fdf7f2 | 211 | void dump(Formatter *f) const; |
181888fb | 212 | }; |
11fdf7f2 | 213 | WRITE_CLASS_ENCODER(LCFilter) |
181888fb | 214 | |
7c673cae FG |
215 | class LCRule |
216 | { | |
217 | protected: | |
218 | string id; | |
219 | string prefix; | |
220 | string status; | |
221 | LCExpiration expiration; | |
222 | LCExpiration noncur_expiration; | |
223 | LCExpiration mp_expiration; | |
181888fb | 224 | LCFilter filter; |
11fdf7f2 TL |
225 | map<string, LCTransition> transitions; |
226 | map<string, LCTransition> noncur_transitions; | |
224ce89b | 227 | bool dm_expiration = false; |
7c673cae FG |
228 | |
229 | public: | |
230 | ||
231 | LCRule(){}; | |
232 | ~LCRule(){}; | |
233 | ||
11fdf7f2 TL |
234 | const string& get_id() const { |
235 | return id; | |
7c673cae FG |
236 | } |
237 | ||
11fdf7f2 | 238 | const string& get_status() const { |
7c673cae FG |
239 | return status; |
240 | } | |
181888fb | 241 | |
11fdf7f2 TL |
242 | bool is_enabled() const { |
243 | return status == "Enabled"; | |
244 | } | |
245 | ||
246 | void set_enabled(bool flag) { | |
247 | status = (flag ? "Enabled" : "Disabled"); | |
248 | } | |
249 | ||
250 | const string& get_prefix() const { | |
7c673cae FG |
251 | return prefix; |
252 | } | |
253 | ||
11fdf7f2 | 254 | const LCFilter& get_filter() const { |
181888fb FG |
255 | return filter; |
256 | } | |
257 | ||
11fdf7f2 | 258 | const LCExpiration& get_expiration() const { |
7c673cae FG |
259 | return expiration; |
260 | } | |
261 | ||
11fdf7f2 | 262 | const LCExpiration& get_noncur_expiration() const { |
7c673cae FG |
263 | return noncur_expiration; |
264 | } | |
265 | ||
11fdf7f2 | 266 | const LCExpiration& get_mp_expiration() const { |
7c673cae FG |
267 | return mp_expiration; |
268 | } | |
269 | ||
11fdf7f2 | 270 | bool get_dm_expiration() const { |
31f18b77 FG |
271 | return dm_expiration; |
272 | } | |
273 | ||
11fdf7f2 TL |
274 | const map<string, LCTransition>& get_transitions() const { |
275 | return transitions; | |
276 | } | |
277 | ||
278 | const map<string, LCTransition>& get_noncur_transitions() const { | |
279 | return noncur_transitions; | |
7c673cae FG |
280 | } |
281 | ||
11fdf7f2 TL |
282 | void set_id(const string& _id) { |
283 | id = _id; | |
284 | } | |
285 | ||
286 | void set_prefix(const string& _prefix) { | |
287 | prefix = _prefix; | |
7c673cae FG |
288 | } |
289 | ||
11fdf7f2 TL |
290 | void set_status(const string& _status) { |
291 | status = _status; | |
7c673cae FG |
292 | } |
293 | ||
11fdf7f2 TL |
294 | void set_expiration(const LCExpiration& _expiration) { |
295 | expiration = _expiration; | |
7c673cae FG |
296 | } |
297 | ||
11fdf7f2 TL |
298 | void set_noncur_expiration(const LCExpiration& _noncur_expiration) { |
299 | noncur_expiration = _noncur_expiration; | |
7c673cae FG |
300 | } |
301 | ||
11fdf7f2 TL |
302 | void set_mp_expiration(const LCExpiration& _mp_expiration) { |
303 | mp_expiration = _mp_expiration; | |
7c673cae FG |
304 | } |
305 | ||
31f18b77 FG |
306 | void set_dm_expiration(bool _dm_expiration) { |
307 | dm_expiration = _dm_expiration; | |
308 | } | |
309 | ||
11fdf7f2 TL |
310 | bool add_transition(const LCTransition& _transition) { |
311 | auto ret = transitions.emplace(_transition.get_storage_class(), _transition); | |
312 | return ret.second; | |
313 | } | |
314 | ||
315 | bool add_noncur_transition(const LCTransition& _noncur_transition) { | |
316 | auto ret = noncur_transitions.emplace(_noncur_transition.get_storage_class(), _noncur_transition); | |
317 | return ret.second; | |
318 | } | |
319 | ||
320 | bool valid() const; | |
7c673cae FG |
321 | |
322 | void encode(bufferlist& bl) const { | |
11fdf7f2 TL |
323 | ENCODE_START(6, 1, bl); |
324 | encode(id, bl); | |
325 | encode(prefix, bl); | |
326 | encode(status, bl); | |
327 | encode(expiration, bl); | |
328 | encode(noncur_expiration, bl); | |
329 | encode(mp_expiration, bl); | |
330 | encode(dm_expiration, bl); | |
331 | encode(filter, bl); | |
332 | encode(transitions, bl); | |
333 | encode(noncur_transitions, bl); | |
7c673cae FG |
334 | ENCODE_FINISH(bl); |
335 | } | |
11fdf7f2 TL |
336 | void decode(bufferlist::const_iterator& bl) { |
337 | DECODE_START_LEGACY_COMPAT_LEN(6, 1, 1, bl); | |
338 | decode(id, bl); | |
339 | decode(prefix, bl); | |
340 | decode(status, bl); | |
341 | decode(expiration, bl); | |
7c673cae | 342 | if (struct_v >=2) { |
11fdf7f2 | 343 | decode(noncur_expiration, bl); |
7c673cae FG |
344 | } |
345 | if (struct_v >= 3) { | |
11fdf7f2 | 346 | decode(mp_expiration, bl); |
7c673cae | 347 | } |
31f18b77 | 348 | if (struct_v >= 4) { |
11fdf7f2 | 349 | decode(dm_expiration, bl); |
31f18b77 | 350 | } |
181888fb | 351 | if (struct_v >= 5) { |
11fdf7f2 TL |
352 | decode(filter, bl); |
353 | } | |
354 | if (struct_v >= 6) { | |
355 | decode(transitions, bl); | |
356 | decode(noncur_transitions, bl); | |
181888fb | 357 | } |
7c673cae FG |
358 | DECODE_FINISH(bl); |
359 | } | |
11fdf7f2 | 360 | void dump(Formatter *f) const; |
7c673cae | 361 | |
11fdf7f2 | 362 | void init_simple_days_rule(std::string_view _id, std::string_view _prefix, int num_days); |
7c673cae FG |
363 | }; |
364 | WRITE_CLASS_ENCODER(LCRule) | |
365 | ||
11fdf7f2 TL |
366 | struct transition_action |
367 | { | |
368 | int days; | |
369 | boost::optional<ceph::real_time> date; | |
370 | string storage_class; | |
371 | transition_action() : days(0) {} | |
372 | }; | |
373 | ||
494da23a | 374 | /* XXX why not LCRule? */ |
7c673cae FG |
375 | struct lc_op |
376 | { | |
494da23a | 377 | string id; |
11fdf7f2 TL |
378 | bool status{false}; |
379 | bool dm_expiration{false}; | |
380 | int expiration{0}; | |
381 | int noncur_expiration{0}; | |
382 | int mp_expiration{0}; | |
224ce89b | 383 | boost::optional<ceph::real_time> expiration_date; |
11fdf7f2 TL |
384 | boost::optional<RGWObjTags> obj_tags; |
385 | map<string, transition_action> transitions; | |
386 | map<string, transition_action> noncur_transitions; | |
494da23a TL |
387 | |
388 | /* ctors are nice */ | |
389 | lc_op() = delete; | |
390 | ||
391 | lc_op(const std::string id) : id(id) | |
392 | {} | |
393 | ||
11fdf7f2 | 394 | void dump(Formatter *f) const; |
7c673cae FG |
395 | }; |
396 | ||
397 | class RGWLifecycleConfiguration | |
398 | { | |
399 | protected: | |
400 | CephContext *cct; | |
494da23a | 401 | multimap<string, lc_op> prefix_map; |
7c673cae | 402 | multimap<string, LCRule> rule_map; |
11fdf7f2 | 403 | bool _add_rule(const LCRule& rule); |
224ce89b | 404 | bool has_same_action(const lc_op& first, const lc_op& second); |
7c673cae | 405 | public: |
11fdf7f2 | 406 | explicit RGWLifecycleConfiguration(CephContext *_cct) : cct(_cct) {} |
7c673cae FG |
407 | RGWLifecycleConfiguration() : cct(NULL) {} |
408 | ||
409 | void set_ctx(CephContext *ctx) { | |
410 | cct = ctx; | |
411 | } | |
412 | ||
413 | virtual ~RGWLifecycleConfiguration() {} | |
414 | ||
415 | // int get_perm(string& id, int perm_mask); | |
416 | // int get_group_perm(ACLGroupTypeEnum group, int perm_mask); | |
417 | void encode(bufferlist& bl) const { | |
418 | ENCODE_START(1, 1, bl); | |
11fdf7f2 | 419 | encode(rule_map, bl); |
7c673cae FG |
420 | ENCODE_FINISH(bl); |
421 | } | |
11fdf7f2 | 422 | void decode(bufferlist::const_iterator& bl) { |
7c673cae | 423 | DECODE_START_LEGACY_COMPAT_LEN(1, 1, 1, bl); |
11fdf7f2 | 424 | decode(rule_map, bl); |
7c673cae FG |
425 | multimap<string, LCRule>::iterator iter; |
426 | for (iter = rule_map.begin(); iter != rule_map.end(); ++iter) { | |
427 | LCRule& rule = iter->second; | |
11fdf7f2 | 428 | _add_rule(rule); |
7c673cae FG |
429 | } |
430 | DECODE_FINISH(bl); | |
431 | } | |
432 | void dump(Formatter *f) const; | |
11fdf7f2 | 433 | static void generate_test_instances(list<RGWLifecycleConfiguration*>& o); |
7c673cae | 434 | |
11fdf7f2 | 435 | void add_rule(const LCRule& rule); |
7c673cae | 436 | |
11fdf7f2 | 437 | int check_and_add_rule(const LCRule& rule); |
7c673cae | 438 | |
224ce89b | 439 | bool valid(); |
7c673cae FG |
440 | |
441 | multimap<string, LCRule>& get_rule_map() { return rule_map; } | |
494da23a | 442 | multimap<string, lc_op>& get_prefix_map() { return prefix_map; } |
7c673cae FG |
443 | /* |
444 | void create_default(string id, string name) { | |
445 | ACLGrant grant; | |
446 | grant.set_canon(id, name, RGW_PERM_FULL_CONTROL); | |
447 | add_grant(&grant); | |
448 | } | |
449 | */ | |
450 | }; | |
451 | WRITE_CLASS_ENCODER(RGWLifecycleConfiguration) | |
452 | ||
11fdf7f2 | 453 | class RGWLC : public DoutPrefixProvider { |
7c673cae | 454 | CephContext *cct; |
9f95a23c | 455 | rgw::sal::RGWRadosStore *store; |
224ce89b WB |
456 | int max_objs{0}; |
457 | string *obj_names{nullptr}; | |
7c673cae FG |
458 | std::atomic<bool> down_flag = { false }; |
459 | string cookie; | |
460 | ||
e306af50 TL |
461 | public: |
462 | ||
463 | class WorkPool; | |
464 | ||
465 | class LCWorker : public Thread | |
466 | { | |
11fdf7f2 | 467 | const DoutPrefixProvider *dpp; |
7c673cae FG |
468 | CephContext *cct; |
469 | RGWLC *lc; | |
f6b5b4d7 TL |
470 | int ix; |
471 | std::mutex lock; | |
472 | std::condition_variable cond; | |
e306af50 | 473 | WorkPool* workpool{nullptr}; |
7c673cae FG |
474 | |
475 | public: | |
f6b5b4d7 TL |
476 | |
477 | using lock_guard = std::lock_guard<std::mutex>; | |
478 | using unique_lock = std::unique_lock<std::mutex>; | |
479 | ||
480 | LCWorker(const DoutPrefixProvider* dpp, CephContext *_cct, RGWLC *_lc, | |
481 | int ix); | |
e306af50 | 482 | RGWLC* get_lc() { return lc; } |
7c673cae FG |
483 | void *entry() override; |
484 | void stop(); | |
485 | bool should_work(utime_t& now); | |
486 | int schedule_next_start_time(utime_t& start, utime_t& now); | |
e306af50 TL |
487 | ~LCWorker(); |
488 | ||
489 | friend class RGWRados; | |
490 | friend class RGWLC; | |
f6b5b4d7 | 491 | friend class WorkQ; |
e306af50 TL |
492 | }; /* LCWorker */ |
493 | ||
494 | friend class RGWRados; | |
495 | ||
496 | std::vector<std::unique_ptr<RGWLC::LCWorker>> workers; | |
497 | ||
498 | RGWLC() : cct(nullptr), store(nullptr) {} | |
499 | ~RGWLC(); | |
7c673cae | 500 | |
9f95a23c | 501 | void initialize(CephContext *_cct, rgw::sal::RGWRadosStore *_store); |
7c673cae FG |
502 | void finalize(); |
503 | ||
f6b5b4d7 TL |
504 | int process(LCWorker* worker, bool once); |
505 | int process(int index, int max_secs, LCWorker* worker, bool once); | |
506 | bool if_already_run_today(time_t start_date); | |
507 | bool expired_session(time_t started); | |
508 | time_t thread_stop_at(); | |
509 | int list_lc_progress(string& marker, uint32_t max_entries, | |
510 | vector<cls_rgw_lc_entry>&, int& index); | |
e306af50 | 511 | int bucket_lc_prepare(int index, LCWorker* worker); |
f6b5b4d7 TL |
512 | int bucket_lc_process(string& shard_id, LCWorker* worker, time_t stop_at, |
513 | bool once); | |
514 | int bucket_lc_post(int index, int max_lock_sec, | |
515 | cls_rgw_lc_entry& entry, int& result, LCWorker* worker); | |
7c673cae FG |
516 | bool going_down(); |
517 | void start_processor(); | |
518 | void stop_processor(); | |
11fdf7f2 TL |
519 | int set_bucket_config(RGWBucketInfo& bucket_info, |
520 | const map<string, bufferlist>& bucket_attrs, | |
521 | RGWLifecycleConfiguration *config); | |
522 | int remove_bucket_config(RGWBucketInfo& bucket_info, | |
523 | const map<string, bufferlist>& bucket_attrs); | |
524 | ||
525 | CephContext *get_cct() const override { return store->ctx(); } | |
526 | unsigned get_subsys() const; | |
527 | std::ostream& gen_prefix(std::ostream& out) const; | |
7c673cae FG |
528 | |
529 | private: | |
11fdf7f2 | 530 | |
494da23a | 531 | int handle_multipart_expiration(RGWRados::Bucket *target, |
e306af50 | 532 | const multimap<string, lc_op>& prefix_map, |
f6b5b4d7 | 533 | LCWorker* worker, time_t stop_at, bool once); |
7c673cae FG |
534 | }; |
535 | ||
11fdf7f2 TL |
536 | namespace rgw::lc { |
537 | ||
9f95a23c | 538 | int fix_lc_shard_entry(rgw::sal::RGWRadosStore *store, const RGWBucketInfo& bucket_info, |
11fdf7f2 | 539 | const map<std::string,bufferlist>& battrs); |
7c673cae | 540 | |
9f95a23c TL |
541 | std::string s3_expiration_header( |
542 | DoutPrefixProvider* dpp, | |
543 | const rgw_obj_key& obj_key, | |
544 | const RGWObjTags& obj_tagset, | |
545 | const ceph::real_time& mtime, | |
546 | const std::map<std::string, buffer::list>& bucket_attrs); | |
547 | ||
f6b5b4d7 TL |
548 | bool s3_multipart_abort_header( |
549 | DoutPrefixProvider* dpp, | |
550 | const rgw_obj_key& obj_key, | |
551 | const ceph::real_time& mtime, | |
552 | const std::map<std::string, buffer::list>& bucket_attrs, | |
553 | ceph::real_time& abort_date, | |
554 | std::string& rule_id); | |
555 | ||
11fdf7f2 | 556 | } // namespace rgw::lc |
7c673cae FG |
557 | |
558 | #endif |