]>
Commit | Line | Data |
---|---|---|
1 | // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- | |
2 | // vim: ts=8 sw=2 smarttab ft=cpp | |
3 | ||
4 | #ifndef CEPH_RGW_LC_H | |
5 | #define CEPH_RGW_LC_H | |
6 | ||
7 | #include <map> | |
8 | #include <string> | |
9 | #include <iostream> | |
10 | ||
11 | #include "common/debug.h" | |
12 | ||
13 | #include "include/types.h" | |
14 | #include "include/rados/librados.hpp" | |
15 | #include "common/ceph_mutex.h" | |
16 | #include "common/Cond.h" | |
17 | #include "common/iso_8601.h" | |
18 | #include "common/Thread.h" | |
19 | #include "rgw_common.h" | |
20 | #include "rgw_rados.h" | |
21 | #include "cls/rgw/cls_rgw_types.h" | |
22 | #include "rgw_tag.h" | |
23 | #include "rgw_sal.h" | |
24 | ||
25 | #include <atomic> | |
26 | #include <tuple> | |
27 | ||
28 | #define HASH_PRIME 7877 | |
29 | #define MAX_ID_LEN 255 | |
30 | static string lc_oid_prefix = "lc"; | |
31 | static string lc_index_lock_name = "lc_process"; | |
32 | ||
33 | extern const char* LC_STATUS[]; | |
34 | ||
35 | typedef enum { | |
36 | lc_uninitial = 0, | |
37 | lc_processing, | |
38 | lc_failed, | |
39 | lc_complete, | |
40 | } LC_BUCKET_STATUS; | |
41 | ||
42 | class LCExpiration | |
43 | { | |
44 | protected: | |
45 | string days; | |
46 | //At present only current object has expiration date | |
47 | string date; | |
48 | public: | |
49 | LCExpiration() {} | |
50 | LCExpiration(const string& _days, const string& _date) : days(_days), date(_date) {} | |
51 | ||
52 | void encode(bufferlist& bl) const { | |
53 | ENCODE_START(3, 2, bl); | |
54 | encode(days, bl); | |
55 | encode(date, bl); | |
56 | ENCODE_FINISH(bl); | |
57 | } | |
58 | void decode(bufferlist::const_iterator& bl) { | |
59 | DECODE_START_LEGACY_COMPAT_LEN(3, 2, 2, bl); | |
60 | decode(days, bl); | |
61 | if (struct_v >= 3) { | |
62 | decode(date, bl); | |
63 | } | |
64 | DECODE_FINISH(bl); | |
65 | } | |
66 | void dump(Formatter *f) const; | |
67 | // static void generate_test_instances(list<ACLOwner*>& o); | |
68 | void set_days(const string& _days) { days = _days; } | |
69 | string get_days_str() const { | |
70 | return days; | |
71 | } | |
72 | int get_days() const {return atoi(days.c_str()); } | |
73 | bool has_days() const { | |
74 | return !days.empty(); | |
75 | } | |
76 | void set_date(const string& _date) { date = _date; } | |
77 | string get_date() const { | |
78 | return date; | |
79 | } | |
80 | bool has_date() const { | |
81 | return !date.empty(); | |
82 | } | |
83 | bool empty() const { | |
84 | return days.empty() && date.empty(); | |
85 | } | |
86 | bool valid() const { | |
87 | if (!days.empty() && !date.empty()) { | |
88 | return false; | |
89 | } else if (!days.empty() && get_days() <= 0) { | |
90 | return false; | |
91 | } | |
92 | //We've checked date in xml parsing | |
93 | return true; | |
94 | } | |
95 | }; | |
96 | WRITE_CLASS_ENCODER(LCExpiration) | |
97 | ||
98 | class LCTransition | |
99 | { | |
100 | protected: | |
101 | string days; | |
102 | string date; | |
103 | string storage_class; | |
104 | ||
105 | public: | |
106 | int get_days() const { | |
107 | return atoi(days.c_str()); | |
108 | } | |
109 | ||
110 | string get_date() const { | |
111 | return date; | |
112 | } | |
113 | ||
114 | string get_storage_class() const { | |
115 | return storage_class; | |
116 | } | |
117 | ||
118 | bool has_days() const { | |
119 | return !days.empty(); | |
120 | } | |
121 | ||
122 | bool has_date() const { | |
123 | return !date.empty(); | |
124 | } | |
125 | ||
126 | bool empty() const { | |
127 | return days.empty() && date.empty(); | |
128 | } | |
129 | ||
130 | bool valid() const { | |
131 | if (!days.empty() && !date.empty()) { | |
132 | return false; | |
133 | } else if (!days.empty() && get_days() < 0) { | |
134 | return false; | |
135 | } | |
136 | //We've checked date in xml parsing | |
137 | return true; | |
138 | } | |
139 | ||
140 | void encode(bufferlist& bl) const { | |
141 | ENCODE_START(1, 1, bl); | |
142 | encode(days, bl); | |
143 | encode(date, bl); | |
144 | encode(storage_class, bl); | |
145 | ENCODE_FINISH(bl); | |
146 | } | |
147 | ||
148 | void decode(bufferlist::const_iterator& bl) { | |
149 | DECODE_START(1, bl); | |
150 | decode(days, bl); | |
151 | decode(date, bl); | |
152 | decode(storage_class, bl); | |
153 | DECODE_FINISH(bl); | |
154 | } | |
155 | void dump(Formatter *f) const; | |
156 | }; | |
157 | WRITE_CLASS_ENCODER(LCTransition) | |
158 | ||
159 | class LCFilter | |
160 | { | |
161 | protected: | |
162 | std::string prefix; | |
163 | RGWObjTags obj_tags; | |
164 | ||
165 | public: | |
166 | ||
167 | const std::string& get_prefix() const { | |
168 | return prefix; | |
169 | } | |
170 | ||
171 | const RGWObjTags& get_tags() const { | |
172 | return obj_tags; | |
173 | } | |
174 | ||
175 | bool empty() const { | |
176 | return !(has_prefix() || has_tags()); | |
177 | } | |
178 | ||
179 | // Determine if we need AND tag when creating xml | |
180 | bool has_multi_condition() const { | |
181 | if (obj_tags.count() > 1) | |
182 | return true; | |
183 | else if (has_prefix() && has_tags()) | |
184 | return true; | |
185 | ||
186 | return false; | |
187 | } | |
188 | ||
189 | bool has_prefix() const { | |
190 | return !prefix.empty(); | |
191 | } | |
192 | ||
193 | bool has_tags() const { | |
194 | return !obj_tags.empty(); | |
195 | } | |
196 | ||
197 | void encode(bufferlist& bl) const { | |
198 | ENCODE_START(2, 1, bl); | |
199 | encode(prefix, bl); | |
200 | encode(obj_tags, bl); | |
201 | ENCODE_FINISH(bl); | |
202 | } | |
203 | void decode(bufferlist::const_iterator& bl) { | |
204 | DECODE_START(2, bl); | |
205 | decode(prefix, bl); | |
206 | if (struct_v >= 2) { | |
207 | decode(obj_tags, bl); | |
208 | } | |
209 | DECODE_FINISH(bl); | |
210 | } | |
211 | void dump(Formatter *f) const; | |
212 | }; | |
213 | WRITE_CLASS_ENCODER(LCFilter) | |
214 | ||
215 | class LCRule | |
216 | { | |
217 | protected: | |
218 | string id; | |
219 | string prefix; | |
220 | string status; | |
221 | LCExpiration expiration; | |
222 | LCExpiration noncur_expiration; | |
223 | LCExpiration mp_expiration; | |
224 | LCFilter filter; | |
225 | map<string, LCTransition> transitions; | |
226 | map<string, LCTransition> noncur_transitions; | |
227 | bool dm_expiration = false; | |
228 | ||
229 | public: | |
230 | ||
231 | LCRule(){}; | |
232 | ~LCRule(){}; | |
233 | ||
234 | const string& get_id() const { | |
235 | return id; | |
236 | } | |
237 | ||
238 | const string& get_status() const { | |
239 | return status; | |
240 | } | |
241 | ||
242 | bool is_enabled() const { | |
243 | return status == "Enabled"; | |
244 | } | |
245 | ||
246 | void set_enabled(bool flag) { | |
247 | status = (flag ? "Enabled" : "Disabled"); | |
248 | } | |
249 | ||
250 | const string& get_prefix() const { | |
251 | return prefix; | |
252 | } | |
253 | ||
254 | const LCFilter& get_filter() const { | |
255 | return filter; | |
256 | } | |
257 | ||
258 | const LCExpiration& get_expiration() const { | |
259 | return expiration; | |
260 | } | |
261 | ||
262 | const LCExpiration& get_noncur_expiration() const { | |
263 | return noncur_expiration; | |
264 | } | |
265 | ||
266 | const LCExpiration& get_mp_expiration() const { | |
267 | return mp_expiration; | |
268 | } | |
269 | ||
270 | bool get_dm_expiration() const { | |
271 | return dm_expiration; | |
272 | } | |
273 | ||
274 | const map<string, LCTransition>& get_transitions() const { | |
275 | return transitions; | |
276 | } | |
277 | ||
278 | const map<string, LCTransition>& get_noncur_transitions() const { | |
279 | return noncur_transitions; | |
280 | } | |
281 | ||
282 | void set_id(const string& _id) { | |
283 | id = _id; | |
284 | } | |
285 | ||
286 | void set_prefix(const string& _prefix) { | |
287 | prefix = _prefix; | |
288 | } | |
289 | ||
290 | void set_status(const string& _status) { | |
291 | status = _status; | |
292 | } | |
293 | ||
294 | void set_expiration(const LCExpiration& _expiration) { | |
295 | expiration = _expiration; | |
296 | } | |
297 | ||
298 | void set_noncur_expiration(const LCExpiration& _noncur_expiration) { | |
299 | noncur_expiration = _noncur_expiration; | |
300 | } | |
301 | ||
302 | void set_mp_expiration(const LCExpiration& _mp_expiration) { | |
303 | mp_expiration = _mp_expiration; | |
304 | } | |
305 | ||
306 | void set_dm_expiration(bool _dm_expiration) { | |
307 | dm_expiration = _dm_expiration; | |
308 | } | |
309 | ||
310 | bool add_transition(const LCTransition& _transition) { | |
311 | auto ret = transitions.emplace(_transition.get_storage_class(), _transition); | |
312 | return ret.second; | |
313 | } | |
314 | ||
315 | bool add_noncur_transition(const LCTransition& _noncur_transition) { | |
316 | auto ret = noncur_transitions.emplace(_noncur_transition.get_storage_class(), _noncur_transition); | |
317 | return ret.second; | |
318 | } | |
319 | ||
320 | bool valid() const; | |
321 | ||
322 | void encode(bufferlist& bl) const { | |
323 | ENCODE_START(6, 1, bl); | |
324 | encode(id, bl); | |
325 | encode(prefix, bl); | |
326 | encode(status, bl); | |
327 | encode(expiration, bl); | |
328 | encode(noncur_expiration, bl); | |
329 | encode(mp_expiration, bl); | |
330 | encode(dm_expiration, bl); | |
331 | encode(filter, bl); | |
332 | encode(transitions, bl); | |
333 | encode(noncur_transitions, bl); | |
334 | ENCODE_FINISH(bl); | |
335 | } | |
336 | void decode(bufferlist::const_iterator& bl) { | |
337 | DECODE_START_LEGACY_COMPAT_LEN(6, 1, 1, bl); | |
338 | decode(id, bl); | |
339 | decode(prefix, bl); | |
340 | decode(status, bl); | |
341 | decode(expiration, bl); | |
342 | if (struct_v >=2) { | |
343 | decode(noncur_expiration, bl); | |
344 | } | |
345 | if (struct_v >= 3) { | |
346 | decode(mp_expiration, bl); | |
347 | } | |
348 | if (struct_v >= 4) { | |
349 | decode(dm_expiration, bl); | |
350 | } | |
351 | if (struct_v >= 5) { | |
352 | decode(filter, bl); | |
353 | } | |
354 | if (struct_v >= 6) { | |
355 | decode(transitions, bl); | |
356 | decode(noncur_transitions, bl); | |
357 | } | |
358 | DECODE_FINISH(bl); | |
359 | } | |
360 | void dump(Formatter *f) const; | |
361 | ||
362 | void init_simple_days_rule(std::string_view _id, std::string_view _prefix, int num_days); | |
363 | }; | |
364 | WRITE_CLASS_ENCODER(LCRule) | |
365 | ||
366 | struct transition_action | |
367 | { | |
368 | int days; | |
369 | boost::optional<ceph::real_time> date; | |
370 | string storage_class; | |
371 | transition_action() : days(0) {} | |
372 | }; | |
373 | ||
374 | /* XXX why not LCRule? */ | |
375 | struct lc_op | |
376 | { | |
377 | string id; | |
378 | bool status{false}; | |
379 | bool dm_expiration{false}; | |
380 | int expiration{0}; | |
381 | int noncur_expiration{0}; | |
382 | int mp_expiration{0}; | |
383 | boost::optional<ceph::real_time> expiration_date; | |
384 | boost::optional<RGWObjTags> obj_tags; | |
385 | map<string, transition_action> transitions; | |
386 | map<string, transition_action> noncur_transitions; | |
387 | ||
388 | /* ctors are nice */ | |
389 | lc_op() = delete; | |
390 | ||
391 | lc_op(const std::string id) : id(id) | |
392 | {} | |
393 | ||
394 | void dump(Formatter *f) const; | |
395 | }; | |
396 | ||
397 | class RGWLifecycleConfiguration | |
398 | { | |
399 | protected: | |
400 | CephContext *cct; | |
401 | multimap<string, lc_op> prefix_map; | |
402 | multimap<string, LCRule> rule_map; | |
403 | bool _add_rule(const LCRule& rule); | |
404 | bool has_same_action(const lc_op& first, const lc_op& second); | |
405 | public: | |
406 | explicit RGWLifecycleConfiguration(CephContext *_cct) : cct(_cct) {} | |
407 | RGWLifecycleConfiguration() : cct(NULL) {} | |
408 | ||
409 | void set_ctx(CephContext *ctx) { | |
410 | cct = ctx; | |
411 | } | |
412 | ||
413 | virtual ~RGWLifecycleConfiguration() {} | |
414 | ||
415 | // int get_perm(string& id, int perm_mask); | |
416 | // int get_group_perm(ACLGroupTypeEnum group, int perm_mask); | |
417 | void encode(bufferlist& bl) const { | |
418 | ENCODE_START(1, 1, bl); | |
419 | encode(rule_map, bl); | |
420 | ENCODE_FINISH(bl); | |
421 | } | |
422 | void decode(bufferlist::const_iterator& bl) { | |
423 | DECODE_START_LEGACY_COMPAT_LEN(1, 1, 1, bl); | |
424 | decode(rule_map, bl); | |
425 | multimap<string, LCRule>::iterator iter; | |
426 | for (iter = rule_map.begin(); iter != rule_map.end(); ++iter) { | |
427 | LCRule& rule = iter->second; | |
428 | _add_rule(rule); | |
429 | } | |
430 | DECODE_FINISH(bl); | |
431 | } | |
432 | void dump(Formatter *f) const; | |
433 | static void generate_test_instances(list<RGWLifecycleConfiguration*>& o); | |
434 | ||
435 | void add_rule(const LCRule& rule); | |
436 | ||
437 | int check_and_add_rule(const LCRule& rule); | |
438 | ||
439 | bool valid(); | |
440 | ||
441 | multimap<string, LCRule>& get_rule_map() { return rule_map; } | |
442 | multimap<string, lc_op>& get_prefix_map() { return prefix_map; } | |
443 | /* | |
444 | void create_default(string id, string name) { | |
445 | ACLGrant grant; | |
446 | grant.set_canon(id, name, RGW_PERM_FULL_CONTROL); | |
447 | add_grant(&grant); | |
448 | } | |
449 | */ | |
450 | }; | |
451 | WRITE_CLASS_ENCODER(RGWLifecycleConfiguration) | |
452 | ||
453 | class RGWLC : public DoutPrefixProvider { | |
454 | CephContext *cct; | |
455 | rgw::sal::RGWRadosStore *store; | |
456 | int max_objs{0}; | |
457 | string *obj_names{nullptr}; | |
458 | std::atomic<bool> down_flag = { false }; | |
459 | string cookie; | |
460 | ||
461 | public: | |
462 | ||
463 | class WorkPool; | |
464 | ||
465 | class LCWorker : public Thread | |
466 | { | |
467 | const DoutPrefixProvider *dpp; | |
468 | CephContext *cct; | |
469 | RGWLC *lc; | |
470 | ceph::mutex lock = ceph::make_mutex("LCWorker"); | |
471 | ceph::condition_variable cond; | |
472 | WorkPool* workpool{nullptr}; | |
473 | ||
474 | public: | |
475 | LCWorker(const DoutPrefixProvider* _dpp, CephContext *_cct, RGWLC *_lc); | |
476 | RGWLC* get_lc() { return lc; } | |
477 | void *entry() override; | |
478 | void stop(); | |
479 | bool should_work(utime_t& now); | |
480 | int schedule_next_start_time(utime_t& start, utime_t& now); | |
481 | ~LCWorker(); | |
482 | ||
483 | friend class RGWRados; | |
484 | friend class RGWLC; | |
485 | }; /* LCWorker */ | |
486 | ||
487 | friend class RGWRados; | |
488 | ||
489 | std::vector<std::unique_ptr<RGWLC::LCWorker>> workers; | |
490 | ||
491 | RGWLC() : cct(nullptr), store(nullptr) {} | |
492 | ~RGWLC(); | |
493 | ||
494 | void initialize(CephContext *_cct, rgw::sal::RGWRadosStore *_store); | |
495 | void finalize(); | |
496 | ||
497 | int process(LCWorker* worker); | |
498 | int process(int index, int max_secs, LCWorker* worker); | |
499 | bool if_already_run_today(time_t& start_date); | |
500 | int list_lc_progress(const string& marker, uint32_t max_entries, map<string, int> *progress_map); | |
501 | int bucket_lc_prepare(int index, LCWorker* worker); | |
502 | int bucket_lc_process(string& shard_id, LCWorker* worker); | |
503 | int bucket_lc_post(int index, int max_lock_sec, pair<string, int >& entry, int& result, LCWorker* worker); | |
504 | bool going_down(); | |
505 | void start_processor(); | |
506 | void stop_processor(); | |
507 | int set_bucket_config(RGWBucketInfo& bucket_info, | |
508 | const map<string, bufferlist>& bucket_attrs, | |
509 | RGWLifecycleConfiguration *config); | |
510 | int remove_bucket_config(RGWBucketInfo& bucket_info, | |
511 | const map<string, bufferlist>& bucket_attrs); | |
512 | ||
513 | CephContext *get_cct() const override { return store->ctx(); } | |
514 | unsigned get_subsys() const; | |
515 | std::ostream& gen_prefix(std::ostream& out) const; | |
516 | ||
517 | private: | |
518 | ||
519 | int handle_multipart_expiration(RGWRados::Bucket *target, | |
520 | const multimap<string, lc_op>& prefix_map, | |
521 | LCWorker* worker); | |
522 | }; | |
523 | ||
524 | namespace rgw::lc { | |
525 | ||
526 | int fix_lc_shard_entry(rgw::sal::RGWRadosStore *store, const RGWBucketInfo& bucket_info, | |
527 | const map<std::string,bufferlist>& battrs); | |
528 | ||
529 | std::string s3_expiration_header( | |
530 | DoutPrefixProvider* dpp, | |
531 | const rgw_obj_key& obj_key, | |
532 | const RGWObjTags& obj_tagset, | |
533 | const ceph::real_time& mtime, | |
534 | const std::map<std::string, buffer::list>& bucket_attrs); | |
535 | ||
536 | } // namespace rgw::lc | |
537 | ||
538 | #endif |