ceph/src/rgw/rgw_lc.h

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab ft=cpp
   3
   4 #ifndef CEPH_RGW_LC_H
   5 #define CEPH_RGW_LC_H
   6
   7 #include <map>
   8 #include <string>
   9 #include <iostream>
  10
  11 #include "common/debug.h"
  12
  13 #include "include/types.h"
  14 #include "include/rados/librados.hpp"
  15 #include "common/ceph_mutex.h"
  16 #include "common/Cond.h"
  17 #include "common/iso_8601.h"
  18 #include "common/Thread.h"
  19 #include "rgw_common.h"
  20 #include "rgw_rados.h"
  21 #include "cls/rgw/cls_rgw_types.h"
  22 #include "rgw_tag.h"
  23 #include "rgw_sal.h"
  24
  25 #include <atomic>
  26 #include <tuple>
  27
  28 #define HASH_PRIME 7877
  29 #define MAX_ID_LEN 255
  30 static string lc_oid_prefix = "lc";
  31 static string lc_index_lock_name = "lc_process";
  32
  33 extern const char* LC_STATUS[];
  34
  35 typedef enum {
  36   lc_uninitial = 0,
  37   lc_processing,
  38   lc_failed,
  39   lc_complete,
  40 } LC_BUCKET_STATUS;
  41
  42 class LCExpiration
  43 {
  44 protected:
  45   string days;
  46   //At present only current object has expiration date
  47   string date;
  48 public:
  49   LCExpiration() {}
  50   LCExpiration(const string& _days, const string& _date) : days(_days), date(_date) {}
  51
  52   void encode(bufferlist& bl) const {
  53     ENCODE_START(3, 2, bl);
  54     encode(days, bl);
  55     encode(date, bl);
  56     ENCODE_FINISH(bl);
  57   }
  58   void decode(bufferlist::const_iterator& bl) {
  59     DECODE_START_LEGACY_COMPAT_LEN(3, 2, 2, bl);
  60     decode(days, bl);
  61     if (struct_v >= 3) {
  62       decode(date, bl);
  63     }
  64     DECODE_FINISH(bl);
  65   }
  66   void dump(Formatter *f) const;
  67 //  static void generate_test_instances(list<ACLOwner*>& o);
  68   void set_days(const string& _days) { days = _days; }
  69   string get_days_str() const {
  70     return days;
  71   }
  72   int get_days() const {return atoi(days.c_str()); }
  73   bool has_days() const {
  74     return !days.empty();
  75   }
  76   void set_date(const string& _date) { date = _date; }
  77   string get_date() const {
  78     return date;
  79   }
  80   bool has_date() const {
  81     return !date.empty();
  82   }
  83   bool empty() const {
  84     return days.empty() && date.empty();
  85   }
  86   bool valid() const {
  87     if (!days.empty() && !date.empty()) {
  88       return false;
  89     } else if (!days.empty() && get_days() <= 0) {
  90       return false;
  91     }
  92     //We've checked date in xml parsing
  93     return true;
  94   }
  95 };
  96 WRITE_CLASS_ENCODER(LCExpiration)
  97
  98 class LCTransition
  99 {
 100 protected:
 101   string days;
 102   string date;
 103   string storage_class;
 104
 105 public:
 106   int get_days() const {
 107     return atoi(days.c_str());
 108   }
 109
 110   string get_date() const {
 111     return date;
 112   }
 113
 114   string get_storage_class() const {
 115     return storage_class;
 116   }
 117
 118   bool has_days() const {
 119     return !days.empty();
 120   }
 121
 122   bool has_date() const {
 123     return !date.empty();
 124   }
 125
 126   bool empty() const {
 127     return days.empty() && date.empty();
 128   }
 129
 130   bool valid() const {
 131     if (!days.empty() && !date.empty()) {
 132       return false;
 133     } else if (!days.empty() && get_days() < 0) {
 134       return false;
 135     }
 136     //We've checked date in xml parsing
 137     return true;
 138   }
 139
 140   void encode(bufferlist& bl) const {
 141     ENCODE_START(1, 1, bl);
 142     encode(days, bl);
 143     encode(date, bl);
 144     encode(storage_class, bl);
 145     ENCODE_FINISH(bl);
 146   }
 147
 148   void decode(bufferlist::const_iterator& bl) {
 149     DECODE_START(1, bl);
 150     decode(days, bl);
 151     decode(date, bl);
 152     decode(storage_class, bl);
 153     DECODE_FINISH(bl);
 154   }
 155   void dump(Formatter *f) const {
 156     f->dump_string("days", days);
 157     f->dump_string("date", date);
 158     f->dump_string("storage_class", storage_class);
 159   }
 160 };
 161 WRITE_CLASS_ENCODER(LCTransition)
 162
 163 class LCFilter
 164 {
 165  protected:
 166   std::string prefix;
 167   RGWObjTags obj_tags;
 168
 169  public:
 170
 171   const std::string& get_prefix() const {
 172     return prefix;
 173   }
 174
 175   const RGWObjTags& get_tags() const {
 176     return obj_tags;
 177   }
 178
 179   bool empty() const {
 180     return !(has_prefix() || has_tags());
 181   }
 182
 183   // Determine if we need AND tag when creating xml
 184   bool has_multi_condition() const {
 185     if (obj_tags.count() > 1)
 186       return true;
 187     else if (has_prefix() && has_tags())
 188       return true;
 189
 190     return false;
 191   }
 192
 193   bool has_prefix() const {
 194     return !prefix.empty();
 195   }
 196
 197   bool has_tags() const {
 198     return !obj_tags.empty();
 199   }
 200
 201   void encode(bufferlist& bl) const {
 202     ENCODE_START(2, 1, bl);
 203     encode(prefix, bl);
 204     encode(obj_tags, bl);
 205     ENCODE_FINISH(bl);
 206   }
 207   void decode(bufferlist::const_iterator& bl) {
 208     DECODE_START(2, bl);
 209     decode(prefix, bl);
 210     if (struct_v >= 2) {
 211       decode(obj_tags, bl);
 212     }
 213     DECODE_FINISH(bl);
 214   }
 215   void dump(Formatter *f) const;
 216 };
 217 WRITE_CLASS_ENCODER(LCFilter)
 218
 219 class LCRule
 220 {
 221 protected:
 222   string id;
 223   string prefix;
 224   string status;
 225   LCExpiration expiration;
 226   LCExpiration noncur_expiration;
 227   LCExpiration mp_expiration;
 228   LCFilter filter;
 229   map<string, LCTransition> transitions;
 230   map<string, LCTransition> noncur_transitions;
 231   bool dm_expiration = false;
 232
 233 public:
 234
 235   LCRule(){};
 236   ~LCRule(){};
 237
 238   const string& get_id() const {
 239       return id;
 240   }
 241
 242   const string& get_status() const {
 243       return status;
 244   }
 245
 246   bool is_enabled() const {
 247     return status == "Enabled";
 248   }
 249
 250   void set_enabled(bool flag) {
 251     status = (flag ? "Enabled" : "Disabled");
 252   }
 253
 254   const string& get_prefix() const {
 255       return prefix;
 256   }
 257
 258   const LCFilter& get_filter() const {
 259     return filter;
 260   }
 261
 262   const LCExpiration& get_expiration() const {
 263     return expiration;
 264   }
 265
 266   const LCExpiration& get_noncur_expiration() const {
 267     return noncur_expiration;
 268   }
 269
 270   const LCExpiration& get_mp_expiration() const {
 271     return mp_expiration;
 272   }
 273
 274   bool get_dm_expiration() const {
 275     return dm_expiration;
 276   }
 277
 278   const map<string, LCTransition>& get_transitions() const {
 279     return transitions;
 280   }
 281
 282   const map<string, LCTransition>& get_noncur_transitions() const {
 283     return noncur_transitions;
 284   }
 285
 286   void set_id(const string& _id) {
 287     id = _id;
 288   }
 289
 290   void set_prefix(const string& _prefix) {
 291     prefix = _prefix;
 292   }
 293
 294   void set_status(const string& _status) {
 295     status = _status;
 296   }
 297
 298   void set_expiration(const LCExpiration& _expiration) {
 299     expiration = _expiration;
 300   }
 301
 302   void set_noncur_expiration(const LCExpiration& _noncur_expiration) {
 303     noncur_expiration = _noncur_expiration;
 304   }
 305
 306   void set_mp_expiration(const LCExpiration& _mp_expiration) {
 307     mp_expiration = _mp_expiration;
 308   }
 309
 310   void set_dm_expiration(bool _dm_expiration) {
 311     dm_expiration = _dm_expiration;
 312   }
 313
 314   bool add_transition(const LCTransition& _transition) {
 315     auto ret = transitions.emplace(_transition.get_storage_class(), _transition);
 316     return ret.second;
 317   }
 318
 319   bool add_noncur_transition(const LCTransition& _noncur_transition) {
 320     auto ret = noncur_transitions.emplace(_noncur_transition.get_storage_class(), _noncur_transition);
 321     return ret.second;
 322   }
 323
 324   bool valid() const;
 325
 326   void encode(bufferlist& bl) const {
 327      ENCODE_START(6, 1, bl);
 328      encode(id, bl);
 329      encode(prefix, bl);
 330      encode(status, bl);
 331      encode(expiration, bl);
 332      encode(noncur_expiration, bl);
 333      encode(mp_expiration, bl);
 334      encode(dm_expiration, bl);
 335      encode(filter, bl);
 336      encode(transitions, bl);
 337      encode(noncur_transitions, bl);
 338      ENCODE_FINISH(bl);
 339    }
 340    void decode(bufferlist::const_iterator& bl) {
 341      DECODE_START_LEGACY_COMPAT_LEN(6, 1, 1, bl);
 342      decode(id, bl);
 343      decode(prefix, bl);
 344      decode(status, bl);
 345      decode(expiration, bl);
 346      if (struct_v >=2) {
 347        decode(noncur_expiration, bl);
 348      }
 349      if (struct_v >= 3) {
 350        decode(mp_expiration, bl);
 351      }
 352      if (struct_v >= 4) {
 353         decode(dm_expiration, bl);
 354      }
 355      if (struct_v >= 5) {
 356        decode(filter, bl);
 357      }
 358      if (struct_v >= 6) {
 359        decode(transitions, bl);
 360        decode(noncur_transitions, bl);
 361      }
 362      DECODE_FINISH(bl);
 363    }
 364   void dump(Formatter *f) const;
 365
 366   void init_simple_days_rule(std::string_view _id, std::string_view _prefix, int num_days);
 367 };
 368 WRITE_CLASS_ENCODER(LCRule)
 369
 370 struct transition_action
 371 {
 372   int days;
 373   boost::optional<ceph::real_time> date;
 374   string storage_class;
 375   transition_action() : days(0) {}
 376   void dump(Formatter *f) const {
 377     if (!date) {
 378       f->dump_int("days", days);
 379     } else {
 380       utime_t ut(*date);
 381       f->dump_stream("date") << ut;
 382     }
 383   }
 384 };
 385
 386 /* XXX why not LCRule? */
 387 struct lc_op
 388 {
 389   string id;
 390   bool status{false};
 391   bool dm_expiration{false};
 392   int expiration{0};
 393   int noncur_expiration{0};
 394   int mp_expiration{0};
 395   boost::optional<ceph::real_time> expiration_date;
 396   boost::optional<RGWObjTags> obj_tags;
 397   map<string, transition_action> transitions;
 398   map<string, transition_action> noncur_transitions;
 399
 400   /* ctors are nice */
 401   lc_op() = delete;
 402
 403   lc_op(const std::string id) : id(id)
 404     {}
 405
 406   void dump(Formatter *f) const;
 407 };
 408
 409 class RGWLifecycleConfiguration
 410 {
 411 protected:
 412   CephContext *cct;
 413   multimap<string, lc_op> prefix_map;
 414   multimap<string, LCRule> rule_map;
 415   bool _add_rule(const LCRule& rule);
 416   bool has_same_action(const lc_op& first, const lc_op& second);
 417 public:
 418   explicit RGWLifecycleConfiguration(CephContext *_cct) : cct(_cct) {}
 419   RGWLifecycleConfiguration() : cct(NULL) {}
 420
 421   void set_ctx(CephContext *ctx) {
 422     cct = ctx;
 423   }
 424
 425   virtual ~RGWLifecycleConfiguration() {}
 426
 427 //  int get_perm(string& id, int perm_mask);
 428 //  int get_group_perm(ACLGroupTypeEnum group, int perm_mask);
 429   void encode(bufferlist& bl) const {
 430     ENCODE_START(1, 1, bl);
 431     encode(rule_map, bl);
 432     ENCODE_FINISH(bl);
 433   }
 434   void decode(bufferlist::const_iterator& bl) {
 435     DECODE_START_LEGACY_COMPAT_LEN(1, 1, 1, bl);
 436     decode(rule_map, bl);
 437     multimap<string, LCRule>::iterator iter;
 438     for (iter = rule_map.begin(); iter != rule_map.end(); ++iter) {
 439       LCRule& rule = iter->second;
 440       _add_rule(rule);
 441     }
 442     DECODE_FINISH(bl);
 443   }
 444   void dump(Formatter *f) const;
 445   static void generate_test_instances(list<RGWLifecycleConfiguration*>& o);
 446
 447   void add_rule(const LCRule& rule);
 448
 449   int check_and_add_rule(const LCRule& rule);
 450
 451   bool valid();
 452
 453   multimap<string, LCRule>& get_rule_map() { return rule_map; }
 454   multimap<string, lc_op>& get_prefix_map() { return prefix_map; }
 455 /*
 456   void create_default(string id, string name) {
 457     ACLGrant grant;
 458     grant.set_canon(id, name, RGW_PERM_FULL_CONTROL);
 459     add_grant(&grant);
 460   }
 461 */
 462 };
 463 WRITE_CLASS_ENCODER(RGWLifecycleConfiguration)
 464
 465 class RGWLC : public DoutPrefixProvider {
 466   CephContext *cct;
 467   rgw::sal::RGWRadosStore *store;
 468   int max_objs{0};
 469   string *obj_names{nullptr};
 470   std::atomic<bool> down_flag = { false };
 471   string cookie;
 472
 473 public:
 474
 475   class WorkPool;
 476
 477   class LCWorker : public Thread
 478   {
 479     const DoutPrefixProvider *dpp;
 480     CephContext *cct;
 481     RGWLC *lc;
 482     int ix;
 483     std::mutex lock;
 484     std::condition_variable cond;
 485     WorkPool* workpool{nullptr};
 486
 487   public:
 488
 489     using lock_guard = std::lock_guard<std::mutex>;
 490     using unique_lock = std::unique_lock<std::mutex>;
 491
 492     LCWorker(const DoutPrefixProvider* dpp, CephContext *_cct, RGWLC *_lc,
 493              int ix);
 494     RGWLC* get_lc() { return lc; }
 495     void *entry() override;
 496     void stop();
 497     bool should_work(utime_t& now);
 498     int schedule_next_start_time(utime_t& start, utime_t& now);
 499     ~LCWorker();
 500
 501     friend class RGWRados;
 502     friend class RGWLC;
 503     friend class WorkQ;
 504   }; /* LCWorker */
 505
 506   friend class RGWRados;
 507
 508   std::vector<std::unique_ptr<RGWLC::LCWorker>> workers;
 509
 510   RGWLC() : cct(nullptr), store(nullptr) {}
 511   ~RGWLC();
 512
 513   void initialize(CephContext *_cct, rgw::sal::RGWRadosStore *_store);
 514   void finalize();
 515
 516   int process(LCWorker* worker, bool once);
 517   int process(int index, int max_secs, LCWorker* worker, bool once);
 518   bool if_already_run_today(time_t start_date);
 519   bool expired_session(time_t started);
 520   time_t thread_stop_at();
 521   int list_lc_progress(string& marker, uint32_t max_entries,
 522                        vector<cls_rgw_lc_entry>&, int& index);
 523   int bucket_lc_prepare(int index, LCWorker* worker);
 524   int bucket_lc_process(string& shard_id, LCWorker* worker, time_t stop_at,
 525                         bool once);
 526   int bucket_lc_post(int index, int max_lock_sec,
 527                      cls_rgw_lc_entry& entry, int& result, LCWorker* worker);
 528   bool going_down();
 529   void start_processor();
 530   void stop_processor();
 531   int set_bucket_config(RGWBucketInfo& bucket_info,
 532                         const map<string, bufferlist>& bucket_attrs,
 533                         RGWLifecycleConfiguration *config);
 534   int remove_bucket_config(RGWBucketInfo& bucket_info,
 535                            const map<string, bufferlist>& bucket_attrs);
 536
 537   CephContext *get_cct() const override { return store->ctx(); }
 538   unsigned get_subsys() const;
 539   std::ostream& gen_prefix(std::ostream& out) const;
 540
 541   private:
 542
 543   int handle_multipart_expiration(RGWRados::Bucket *target,
 544                                   const multimap<string, lc_op>& prefix_map,
 545                                   LCWorker* worker, time_t stop_at, bool once);
 546 };
 547
 548 namespace rgw::lc {
 549
 550 int fix_lc_shard_entry(rgw::sal::RGWRadosStore *store, const RGWBucketInfo& bucket_info,
 551                        const map<std::string,bufferlist>& battrs);
 552
 553 std::string s3_expiration_header(
 554   DoutPrefixProvider* dpp,
 555   const rgw_obj_key& obj_key,
 556   const RGWObjTags& obj_tagset,
 557   const ceph::real_time& mtime,
 558   const std::map<std::string, buffer::list>& bucket_attrs);
 559
 560 bool s3_multipart_abort_header(
 561   DoutPrefixProvider* dpp,
 562   const rgw_obj_key& obj_key,
 563   const ceph::real_time& mtime,
 564   const std::map<std::string, buffer::list>& bucket_attrs,
 565   ceph::real_time& abort_date,
 566   std::string& rule_id);
 567
 568 } // namespace rgw::lc
 569
 570 #endif