]> git.proxmox.com Git - ceph.git/blob - ceph/src/rgw/rgw_lc.h
60f3863fc837e942b03a89398e16cb576714b29e
[ceph.git] / ceph / src / rgw / rgw_lc.h
1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab ft=cpp
3
4 #ifndef CEPH_RGW_LC_H
5 #define CEPH_RGW_LC_H
6
7 #include <map>
8 #include <string>
9 #include <iostream>
10
11 #include "common/debug.h"
12
13 #include "include/types.h"
14 #include "include/rados/librados.hpp"
15 #include "common/ceph_mutex.h"
16 #include "common/Cond.h"
17 #include "common/iso_8601.h"
18 #include "common/Thread.h"
19 #include "rgw_common.h"
20 #include "rgw_rados.h"
21 #include "cls/rgw/cls_rgw_types.h"
22 #include "rgw_tag.h"
23 #include "rgw_sal.h"
24
25 #include <atomic>
26 #include <tuple>
27
28 #define HASH_PRIME 7877
29 #define MAX_ID_LEN 255
30 static string lc_oid_prefix = "lc";
31 static string lc_index_lock_name = "lc_process";
32
33 extern const char* LC_STATUS[];
34
35 typedef enum {
36 lc_uninitial = 0,
37 lc_processing,
38 lc_failed,
39 lc_complete,
40 } LC_BUCKET_STATUS;
41
42 class LCExpiration
43 {
44 protected:
45 string days;
46 //At present only current object has expiration date
47 string date;
48 public:
49 LCExpiration() {}
50 LCExpiration(const string& _days, const string& _date) : days(_days), date(_date) {}
51
52 void encode(bufferlist& bl) const {
53 ENCODE_START(3, 2, bl);
54 encode(days, bl);
55 encode(date, bl);
56 ENCODE_FINISH(bl);
57 }
58 void decode(bufferlist::const_iterator& bl) {
59 DECODE_START_LEGACY_COMPAT_LEN(3, 2, 2, bl);
60 decode(days, bl);
61 if (struct_v >= 3) {
62 decode(date, bl);
63 }
64 DECODE_FINISH(bl);
65 }
66 void dump(Formatter *f) const;
67 // static void generate_test_instances(list<ACLOwner*>& o);
68 void set_days(const string& _days) { days = _days; }
69 string get_days_str() const {
70 return days;
71 }
72 int get_days() const {return atoi(days.c_str()); }
73 bool has_days() const {
74 return !days.empty();
75 }
76 void set_date(const string& _date) { date = _date; }
77 string get_date() const {
78 return date;
79 }
80 bool has_date() const {
81 return !date.empty();
82 }
83 bool empty() const {
84 return days.empty() && date.empty();
85 }
86 bool valid() const {
87 if (!days.empty() && !date.empty()) {
88 return false;
89 } else if (!days.empty() && get_days() <= 0) {
90 return false;
91 }
92 //We've checked date in xml parsing
93 return true;
94 }
95 };
96 WRITE_CLASS_ENCODER(LCExpiration)
97
98 class LCTransition
99 {
100 protected:
101 string days;
102 string date;
103 string storage_class;
104
105 public:
106 int get_days() const {
107 return atoi(days.c_str());
108 }
109
110 string get_date() const {
111 return date;
112 }
113
114 string get_storage_class() const {
115 return storage_class;
116 }
117
118 bool has_days() const {
119 return !days.empty();
120 }
121
122 bool has_date() const {
123 return !date.empty();
124 }
125
126 bool empty() const {
127 return days.empty() && date.empty();
128 }
129
130 bool valid() const {
131 if (!days.empty() && !date.empty()) {
132 return false;
133 } else if (!days.empty() && get_days() < 0) {
134 return false;
135 }
136 //We've checked date in xml parsing
137 return true;
138 }
139
140 void encode(bufferlist& bl) const {
141 ENCODE_START(1, 1, bl);
142 encode(days, bl);
143 encode(date, bl);
144 encode(storage_class, bl);
145 ENCODE_FINISH(bl);
146 }
147
148 void decode(bufferlist::const_iterator& bl) {
149 DECODE_START(1, bl);
150 decode(days, bl);
151 decode(date, bl);
152 decode(storage_class, bl);
153 DECODE_FINISH(bl);
154 }
155 void dump(Formatter *f) const;
156 };
157 WRITE_CLASS_ENCODER(LCTransition)
158
159 class LCFilter
160 {
161 protected:
162 std::string prefix;
163 RGWObjTags obj_tags;
164
165 public:
166
167 const std::string& get_prefix() const {
168 return prefix;
169 }
170
171 const RGWObjTags& get_tags() const {
172 return obj_tags;
173 }
174
175 bool empty() const {
176 return !(has_prefix() || has_tags());
177 }
178
179 // Determine if we need AND tag when creating xml
180 bool has_multi_condition() const {
181 if (obj_tags.count() > 1)
182 return true;
183 else if (has_prefix() && has_tags())
184 return true;
185
186 return false;
187 }
188
189 bool has_prefix() const {
190 return !prefix.empty();
191 }
192
193 bool has_tags() const {
194 return !obj_tags.empty();
195 }
196
197 void encode(bufferlist& bl) const {
198 ENCODE_START(2, 1, bl);
199 encode(prefix, bl);
200 encode(obj_tags, bl);
201 ENCODE_FINISH(bl);
202 }
203 void decode(bufferlist::const_iterator& bl) {
204 DECODE_START(2, bl);
205 decode(prefix, bl);
206 if (struct_v >= 2) {
207 decode(obj_tags, bl);
208 }
209 DECODE_FINISH(bl);
210 }
211 void dump(Formatter *f) const;
212 };
213 WRITE_CLASS_ENCODER(LCFilter)
214
215 class LCRule
216 {
217 protected:
218 string id;
219 string prefix;
220 string status;
221 LCExpiration expiration;
222 LCExpiration noncur_expiration;
223 LCExpiration mp_expiration;
224 LCFilter filter;
225 map<string, LCTransition> transitions;
226 map<string, LCTransition> noncur_transitions;
227 bool dm_expiration = false;
228
229 public:
230
231 LCRule(){};
232 ~LCRule(){};
233
234 const string& get_id() const {
235 return id;
236 }
237
238 const string& get_status() const {
239 return status;
240 }
241
242 bool is_enabled() const {
243 return status == "Enabled";
244 }
245
246 void set_enabled(bool flag) {
247 status = (flag ? "Enabled" : "Disabled");
248 }
249
250 const string& get_prefix() const {
251 return prefix;
252 }
253
254 const LCFilter& get_filter() const {
255 return filter;
256 }
257
258 const LCExpiration& get_expiration() const {
259 return expiration;
260 }
261
262 const LCExpiration& get_noncur_expiration() const {
263 return noncur_expiration;
264 }
265
266 const LCExpiration& get_mp_expiration() const {
267 return mp_expiration;
268 }
269
270 bool get_dm_expiration() const {
271 return dm_expiration;
272 }
273
274 const map<string, LCTransition>& get_transitions() const {
275 return transitions;
276 }
277
278 const map<string, LCTransition>& get_noncur_transitions() const {
279 return noncur_transitions;
280 }
281
282 void set_id(const string& _id) {
283 id = _id;
284 }
285
286 void set_prefix(const string& _prefix) {
287 prefix = _prefix;
288 }
289
290 void set_status(const string& _status) {
291 status = _status;
292 }
293
294 void set_expiration(const LCExpiration& _expiration) {
295 expiration = _expiration;
296 }
297
298 void set_noncur_expiration(const LCExpiration& _noncur_expiration) {
299 noncur_expiration = _noncur_expiration;
300 }
301
302 void set_mp_expiration(const LCExpiration& _mp_expiration) {
303 mp_expiration = _mp_expiration;
304 }
305
306 void set_dm_expiration(bool _dm_expiration) {
307 dm_expiration = _dm_expiration;
308 }
309
310 bool add_transition(const LCTransition& _transition) {
311 auto ret = transitions.emplace(_transition.get_storage_class(), _transition);
312 return ret.second;
313 }
314
315 bool add_noncur_transition(const LCTransition& _noncur_transition) {
316 auto ret = noncur_transitions.emplace(_noncur_transition.get_storage_class(), _noncur_transition);
317 return ret.second;
318 }
319
320 bool valid() const;
321
322 void encode(bufferlist& bl) const {
323 ENCODE_START(6, 1, bl);
324 encode(id, bl);
325 encode(prefix, bl);
326 encode(status, bl);
327 encode(expiration, bl);
328 encode(noncur_expiration, bl);
329 encode(mp_expiration, bl);
330 encode(dm_expiration, bl);
331 encode(filter, bl);
332 encode(transitions, bl);
333 encode(noncur_transitions, bl);
334 ENCODE_FINISH(bl);
335 }
336 void decode(bufferlist::const_iterator& bl) {
337 DECODE_START_LEGACY_COMPAT_LEN(6, 1, 1, bl);
338 decode(id, bl);
339 decode(prefix, bl);
340 decode(status, bl);
341 decode(expiration, bl);
342 if (struct_v >=2) {
343 decode(noncur_expiration, bl);
344 }
345 if (struct_v >= 3) {
346 decode(mp_expiration, bl);
347 }
348 if (struct_v >= 4) {
349 decode(dm_expiration, bl);
350 }
351 if (struct_v >= 5) {
352 decode(filter, bl);
353 }
354 if (struct_v >= 6) {
355 decode(transitions, bl);
356 decode(noncur_transitions, bl);
357 }
358 DECODE_FINISH(bl);
359 }
360 void dump(Formatter *f) const;
361
362 void init_simple_days_rule(std::string_view _id, std::string_view _prefix, int num_days);
363 };
364 WRITE_CLASS_ENCODER(LCRule)
365
366 struct transition_action
367 {
368 int days;
369 boost::optional<ceph::real_time> date;
370 string storage_class;
371 transition_action() : days(0) {}
372 };
373
374 /* XXX why not LCRule? */
375 struct lc_op
376 {
377 string id;
378 bool status{false};
379 bool dm_expiration{false};
380 int expiration{0};
381 int noncur_expiration{0};
382 int mp_expiration{0};
383 boost::optional<ceph::real_time> expiration_date;
384 boost::optional<RGWObjTags> obj_tags;
385 map<string, transition_action> transitions;
386 map<string, transition_action> noncur_transitions;
387
388 /* ctors are nice */
389 lc_op() = delete;
390
391 lc_op(const std::string id) : id(id)
392 {}
393
394 void dump(Formatter *f) const;
395 };
396
397 class RGWLifecycleConfiguration
398 {
399 protected:
400 CephContext *cct;
401 multimap<string, lc_op> prefix_map;
402 multimap<string, LCRule> rule_map;
403 bool _add_rule(const LCRule& rule);
404 bool has_same_action(const lc_op& first, const lc_op& second);
405 public:
406 explicit RGWLifecycleConfiguration(CephContext *_cct) : cct(_cct) {}
407 RGWLifecycleConfiguration() : cct(NULL) {}
408
409 void set_ctx(CephContext *ctx) {
410 cct = ctx;
411 }
412
413 virtual ~RGWLifecycleConfiguration() {}
414
415 // int get_perm(string& id, int perm_mask);
416 // int get_group_perm(ACLGroupTypeEnum group, int perm_mask);
417 void encode(bufferlist& bl) const {
418 ENCODE_START(1, 1, bl);
419 encode(rule_map, bl);
420 ENCODE_FINISH(bl);
421 }
422 void decode(bufferlist::const_iterator& bl) {
423 DECODE_START_LEGACY_COMPAT_LEN(1, 1, 1, bl);
424 decode(rule_map, bl);
425 multimap<string, LCRule>::iterator iter;
426 for (iter = rule_map.begin(); iter != rule_map.end(); ++iter) {
427 LCRule& rule = iter->second;
428 _add_rule(rule);
429 }
430 DECODE_FINISH(bl);
431 }
432 void dump(Formatter *f) const;
433 static void generate_test_instances(list<RGWLifecycleConfiguration*>& o);
434
435 void add_rule(const LCRule& rule);
436
437 int check_and_add_rule(const LCRule& rule);
438
439 bool valid();
440
441 multimap<string, LCRule>& get_rule_map() { return rule_map; }
442 multimap<string, lc_op>& get_prefix_map() { return prefix_map; }
443 /*
444 void create_default(string id, string name) {
445 ACLGrant grant;
446 grant.set_canon(id, name, RGW_PERM_FULL_CONTROL);
447 add_grant(&grant);
448 }
449 */
450 };
451 WRITE_CLASS_ENCODER(RGWLifecycleConfiguration)
452
453 class RGWLC : public DoutPrefixProvider {
454 CephContext *cct;
455 rgw::sal::RGWRadosStore *store;
456 int max_objs{0};
457 string *obj_names{nullptr};
458 std::atomic<bool> down_flag = { false };
459 string cookie;
460
461 public:
462
463 class WorkPool;
464
465 class LCWorker : public Thread
466 {
467 const DoutPrefixProvider *dpp;
468 CephContext *cct;
469 RGWLC *lc;
470 int ix;
471 std::mutex lock;
472 std::condition_variable cond;
473 WorkPool* workpool{nullptr};
474
475 public:
476
477 using lock_guard = std::lock_guard<std::mutex>;
478 using unique_lock = std::unique_lock<std::mutex>;
479
480 LCWorker(const DoutPrefixProvider* dpp, CephContext *_cct, RGWLC *_lc,
481 int ix);
482 RGWLC* get_lc() { return lc; }
483 void *entry() override;
484 void stop();
485 bool should_work(utime_t& now);
486 int schedule_next_start_time(utime_t& start, utime_t& now);
487 ~LCWorker();
488
489 friend class RGWRados;
490 friend class RGWLC;
491 friend class WorkQ;
492 }; /* LCWorker */
493
494 friend class RGWRados;
495
496 std::vector<std::unique_ptr<RGWLC::LCWorker>> workers;
497
498 RGWLC() : cct(nullptr), store(nullptr) {}
499 ~RGWLC();
500
501 void initialize(CephContext *_cct, rgw::sal::RGWRadosStore *_store);
502 void finalize();
503
504 int process(LCWorker* worker, bool once);
505 int process(int index, int max_secs, LCWorker* worker, bool once);
506 bool if_already_run_today(time_t start_date);
507 bool expired_session(time_t started);
508 time_t thread_stop_at();
509 int list_lc_progress(string& marker, uint32_t max_entries,
510 vector<cls_rgw_lc_entry>&, int& index);
511 int bucket_lc_prepare(int index, LCWorker* worker);
512 int bucket_lc_process(string& shard_id, LCWorker* worker, time_t stop_at,
513 bool once);
514 int bucket_lc_post(int index, int max_lock_sec,
515 cls_rgw_lc_entry& entry, int& result, LCWorker* worker);
516 bool going_down();
517 void start_processor();
518 void stop_processor();
519 int set_bucket_config(RGWBucketInfo& bucket_info,
520 const map<string, bufferlist>& bucket_attrs,
521 RGWLifecycleConfiguration *config);
522 int remove_bucket_config(RGWBucketInfo& bucket_info,
523 const map<string, bufferlist>& bucket_attrs);
524
525 CephContext *get_cct() const override { return store->ctx(); }
526 unsigned get_subsys() const;
527 std::ostream& gen_prefix(std::ostream& out) const;
528
529 private:
530
531 int handle_multipart_expiration(RGWRados::Bucket *target,
532 const multimap<string, lc_op>& prefix_map,
533 LCWorker* worker, time_t stop_at, bool once);
534 };
535
536 namespace rgw::lc {
537
538 int fix_lc_shard_entry(rgw::sal::RGWRadosStore *store, const RGWBucketInfo& bucket_info,
539 const map<std::string,bufferlist>& battrs);
540
541 std::string s3_expiration_header(
542 DoutPrefixProvider* dpp,
543 const rgw_obj_key& obj_key,
544 const RGWObjTags& obj_tagset,
545 const ceph::real_time& mtime,
546 const std::map<std::string, buffer::list>& bucket_attrs);
547
548 bool s3_multipart_abort_header(
549 DoutPrefixProvider* dpp,
550 const rgw_obj_key& obj_key,
551 const ceph::real_time& mtime,
552 const std::map<std::string, buffer::list>& bucket_attrs,
553 ceph::real_time& abort_date,
554 std::string& rule_id);
555
556 } // namespace rgw::lc
557
558 #endif