ceph/src/rgw/rgw_lc.h

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3
   4 #ifndef CEPH_RGW_LC_H
   5 #define CEPH_RGW_LC_H
   6
   7 #include <map>
   8 #include <string>
   9 #include <iostream>
  10
  11 #include "common/debug.h"
  12
  13 #include "include/types.h"
  14 #include "include/rados/librados.hpp"
  15 #include "common/Mutex.h"
  16 #include "common/Cond.h"
  17 #include "common/iso_8601.h"
  18 #include "common/Thread.h"
  19 #include "rgw_common.h"
  20 #include "rgw_rados.h"
  21 #include "rgw_multi.h"
  22 #include "cls/rgw/cls_rgw_types.h"
  23 #include "rgw_tag.h"
  24
  25 #include <atomic>
  26
  27 #define HASH_PRIME 7877
  28 #define MAX_ID_LEN 255
  29 static string lc_oid_prefix = "lc";
  30 static string lc_index_lock_name = "lc_process";
  31
  32 extern const char* LC_STATUS[];
  33
  34 typedef enum {
  35   lc_uninitial = 0,
  36   lc_processing,
  37   lc_failed,
  38   lc_complete,
  39 } LC_BUCKET_STATUS;
  40
  41 class LCExpiration
  42 {
  43 protected:
  44   string days;
  45   //At present only current object has expiration date
  46   string date;
  47 public:
  48   LCExpiration() {}
  49   LCExpiration(const string& _days, const string& _date) : days(_days), date(_date) {}
  50
  51   void encode(bufferlist& bl) const {
  52     ENCODE_START(3, 2, bl);
  53     encode(days, bl);
  54     encode(date, bl);
  55     ENCODE_FINISH(bl);
  56   }
  57   void decode(bufferlist::const_iterator& bl) {
  58     DECODE_START_LEGACY_COMPAT_LEN(3, 2, 2, bl);
  59     decode(days, bl);
  60     if (struct_v >= 3) {
  61       decode(date, bl);
  62     }
  63     DECODE_FINISH(bl);
  64   }
  65   void dump(Formatter *f) const;
  66 //  static void generate_test_instances(list<ACLOwner*>& o);
  67   void set_days(const string& _days) { days = _days; }
  68   string get_days_str() const {
  69     return days;
  70   }
  71   int get_days() const {return atoi(days.c_str()); }
  72   bool has_days() const {
  73     return !days.empty();
  74   }
  75   void set_date(const string& _date) { date = _date; }
  76   string get_date() const {
  77     return date;
  78   }
  79   bool has_date() const {
  80     return !date.empty();
  81   }
  82   bool empty() const {
  83     return days.empty() && date.empty();
  84   }
  85   bool valid() const {
  86     if (!days.empty() && !date.empty()) {
  87       return false;
  88     } else if (!days.empty() && get_days() <= 0) {
  89       return false;
  90     }
  91     //We've checked date in xml parsing
  92     return true;
  93   }
  94 };
  95 WRITE_CLASS_ENCODER(LCExpiration)
  96
  97 class LCTransition
  98 {
  99 protected:
 100   string days;
 101   string date;
 102   string storage_class;
 103
 104 public:
 105   int get_days() const {
 106     return atoi(days.c_str());
 107   }
 108
 109   string get_date() const {
 110     return date;
 111   }
 112
 113   string get_storage_class() const {
 114     return storage_class;
 115   }
 116
 117   bool has_days() const {
 118     return !days.empty();
 119   }
 120
 121   bool has_date() const {
 122     return !date.empty();
 123   }
 124
 125   bool empty() const {
 126     return days.empty() && date.empty();
 127   }
 128
 129   bool valid() const {
 130     if (!days.empty() && !date.empty()) {
 131       return false;
 132     } else if (!days.empty() && get_days() <=0) {
 133       return false;
 134     }
 135     //We've checked date in xml parsing
 136     return true;
 137   }
 138
 139   void encode(bufferlist& bl) const {
 140     ENCODE_START(1, 1, bl);
 141     encode(days, bl);
 142     encode(date, bl);
 143     encode(storage_class, bl);
 144     ENCODE_FINISH(bl);
 145   }
 146
 147   void decode(bufferlist::const_iterator& bl) {
 148     DECODE_START(1, bl);
 149     decode(days, bl);
 150     decode(date, bl);
 151     decode(storage_class, bl);
 152     DECODE_FINISH(bl);
 153   }
 154   void dump(Formatter *f) const;
 155 };
 156 WRITE_CLASS_ENCODER(LCTransition)
 157
 158 class LCFilter
 159 {
 160  protected:
 161   std::string prefix;
 162   RGWObjTags obj_tags;
 163
 164  public:
 165
 166   const std::string& get_prefix() const {
 167     return prefix;
 168   }
 169
 170   const RGWObjTags& get_tags() const {
 171     return obj_tags;
 172   }
 173
 174   bool empty() const {
 175     return !(has_prefix() || has_tags());
 176   }
 177
 178   // Determine if we need AND tag when creating xml
 179   bool has_multi_condition() const {
 180     if (obj_tags.count() > 1)
 181       return true;
 182     else if (has_prefix() && has_tags())
 183       return true;
 184
 185     return false;
 186   }
 187
 188   bool has_prefix() const {
 189     return !prefix.empty();
 190   }
 191
 192   bool has_tags() const {
 193     return !obj_tags.empty();
 194   }
 195
 196   void encode(bufferlist& bl) const {
 197     ENCODE_START(2, 1, bl);
 198     encode(prefix, bl);
 199     encode(obj_tags, bl);
 200     ENCODE_FINISH(bl);
 201   }
 202   void decode(bufferlist::const_iterator& bl) {
 203     DECODE_START(2, bl);
 204     decode(prefix, bl);
 205     if (struct_v >= 2) {
 206       decode(obj_tags, bl);
 207     }
 208     DECODE_FINISH(bl);
 209   }
 210   void dump(Formatter *f) const;
 211 };
 212 WRITE_CLASS_ENCODER(LCFilter)
 213
 214
 215
 216 class LCRule
 217 {
 218 protected:
 219   string id;
 220   string prefix;
 221   string status;
 222   LCExpiration expiration;
 223   LCExpiration noncur_expiration;
 224   LCExpiration mp_expiration;
 225   LCFilter filter;
 226   map<string, LCTransition> transitions;
 227   map<string, LCTransition> noncur_transitions;
 228   bool dm_expiration = false;
 229
 230 public:
 231
 232   LCRule(){};
 233   ~LCRule(){};
 234
 235   const string& get_id() const {
 236       return id;
 237   }
 238
 239   const string& get_status() const {
 240       return status;
 241   }
 242
 243   bool is_enabled() const {
 244     return status == "Enabled";
 245   }
 246
 247   void set_enabled(bool flag) {
 248     status = (flag ? "Enabled" : "Disabled");
 249   }
 250
 251   const string& get_prefix() const {
 252       return prefix;
 253   }
 254
 255   const LCFilter& get_filter() const {
 256     return filter;
 257   }
 258
 259   const LCExpiration& get_expiration() const {
 260     return expiration;
 261   }
 262
 263   const LCExpiration& get_noncur_expiration() const {
 264     return noncur_expiration;
 265   }
 266
 267   const LCExpiration& get_mp_expiration() const {
 268     return mp_expiration;
 269   }
 270
 271   bool get_dm_expiration() const {
 272     return dm_expiration;
 273   }
 274
 275   const map<string, LCTransition>& get_transitions() const {
 276     return transitions;
 277   }
 278
 279   const map<string, LCTransition>& get_noncur_transitions() const {
 280     return noncur_transitions;
 281   }
 282
 283   void set_id(const string& _id) {
 284     id = _id;
 285   }
 286
 287   void set_prefix(const string& _prefix) {
 288     prefix = _prefix;
 289   }
 290
 291   void set_status(const string& _status) {
 292     status = _status;
 293   }
 294
 295   void set_expiration(const LCExpiration& _expiration) {
 296     expiration = _expiration;
 297   }
 298
 299   void set_noncur_expiration(const LCExpiration& _noncur_expiration) {
 300     noncur_expiration = _noncur_expiration;
 301   }
 302
 303   void set_mp_expiration(const LCExpiration& _mp_expiration) {
 304     mp_expiration = _mp_expiration;
 305   }
 306
 307   void set_dm_expiration(bool _dm_expiration) {
 308     dm_expiration = _dm_expiration;
 309   }
 310
 311   bool add_transition(const LCTransition& _transition) {
 312     auto ret = transitions.emplace(_transition.get_storage_class(), _transition);
 313     return ret.second;
 314   }
 315
 316   bool add_noncur_transition(const LCTransition& _noncur_transition) {
 317     auto ret = noncur_transitions.emplace(_noncur_transition.get_storage_class(), _noncur_transition);
 318     return ret.second;
 319   }
 320
 321   bool valid() const;
 322
 323   void encode(bufferlist& bl) const {
 324      ENCODE_START(6, 1, bl);
 325      encode(id, bl);
 326      encode(prefix, bl);
 327      encode(status, bl);
 328      encode(expiration, bl);
 329      encode(noncur_expiration, bl);
 330      encode(mp_expiration, bl);
 331      encode(dm_expiration, bl);
 332      encode(filter, bl);
 333      encode(transitions, bl);
 334      encode(noncur_transitions, bl);
 335      ENCODE_FINISH(bl);
 336    }
 337    void decode(bufferlist::const_iterator& bl) {
 338      DECODE_START_LEGACY_COMPAT_LEN(6, 1, 1, bl);
 339      decode(id, bl);
 340      decode(prefix, bl);
 341      decode(status, bl);
 342      decode(expiration, bl);
 343      if (struct_v >=2) {
 344        decode(noncur_expiration, bl);
 345      }
 346      if (struct_v >= 3) {
 347        decode(mp_expiration, bl);
 348      }
 349      if (struct_v >= 4) {
 350         decode(dm_expiration, bl);
 351      }
 352      if (struct_v >= 5) {
 353        decode(filter, bl);
 354      }
 355      if (struct_v >= 6) {
 356        decode(transitions, bl);
 357        decode(noncur_transitions, bl);
 358      }
 359      DECODE_FINISH(bl);
 360    }
 361   void dump(Formatter *f) const;
 362
 363   void init_simple_days_rule(std::string_view _id, std::string_view _prefix, int num_days);
 364 };
 365 WRITE_CLASS_ENCODER(LCRule)
 366
 367 struct transition_action
 368 {
 369   int days;
 370   boost::optional<ceph::real_time> date;
 371   string storage_class;
 372   transition_action() : days(0) {}
 373 };
 374
 375 /* XXX why not LCRule? */
 376 struct lc_op
 377 {
 378   string id;
 379   bool status{false};
 380   bool dm_expiration{false};
 381   int expiration{0};
 382   int noncur_expiration{0};
 383   int mp_expiration{0};
 384   boost::optional<ceph::real_time> expiration_date;
 385   boost::optional<RGWObjTags> obj_tags;
 386   map<string, transition_action> transitions;
 387   map<string, transition_action> noncur_transitions;
 388
 389   /* ctors are nice */
 390   lc_op() = delete;
 391
 392   lc_op(const std::string id) : id(id)
 393     {}
 394
 395   void dump(Formatter *f) const;
 396 };
 397
 398 class RGWLifecycleConfiguration
 399 {
 400 protected:
 401   CephContext *cct;
 402   multimap<string, lc_op> prefix_map;
 403   multimap<string, LCRule> rule_map;
 404   bool _add_rule(const LCRule& rule);
 405   bool has_same_action(const lc_op& first, const lc_op& second);
 406 public:
 407   explicit RGWLifecycleConfiguration(CephContext *_cct) : cct(_cct) {}
 408   RGWLifecycleConfiguration() : cct(NULL) {}
 409
 410   void set_ctx(CephContext *ctx) {
 411     cct = ctx;
 412   }
 413
 414   virtual ~RGWLifecycleConfiguration() {}
 415
 416 //  int get_perm(string& id, int perm_mask);
 417 //  int get_group_perm(ACLGroupTypeEnum group, int perm_mask);
 418   void encode(bufferlist& bl) const {
 419     ENCODE_START(1, 1, bl);
 420     encode(rule_map, bl);
 421     ENCODE_FINISH(bl);
 422   }
 423   void decode(bufferlist::const_iterator& bl) {
 424     DECODE_START_LEGACY_COMPAT_LEN(1, 1, 1, bl);
 425     decode(rule_map, bl);
 426     multimap<string, LCRule>::iterator iter;
 427     for (iter = rule_map.begin(); iter != rule_map.end(); ++iter) {
 428       LCRule& rule = iter->second;
 429       _add_rule(rule);
 430     }
 431     DECODE_FINISH(bl);
 432   }
 433   void dump(Formatter *f) const;
 434   static void generate_test_instances(list<RGWLifecycleConfiguration*>& o);
 435
 436   void add_rule(const LCRule& rule);
 437
 438   int check_and_add_rule(const LCRule& rule);
 439
 440   bool valid();
 441
 442   multimap<string, LCRule>& get_rule_map() { return rule_map; }
 443   multimap<string, lc_op>& get_prefix_map() { return prefix_map; }
 444 /*
 445   void create_default(string id, string name) {
 446     ACLGrant grant;
 447     grant.set_canon(id, name, RGW_PERM_FULL_CONTROL);
 448     add_grant(&grant);
 449   }
 450 */
 451 };
 452 WRITE_CLASS_ENCODER(RGWLifecycleConfiguration)
 453
 454 class RGWLC : public DoutPrefixProvider {
 455   CephContext *cct;
 456   RGWRados *store;
 457   int max_objs{0};
 458   string *obj_names{nullptr};
 459   std::atomic<bool> down_flag = { false };
 460   string cookie;
 461
 462   class LCWorker : public Thread {
 463     const DoutPrefixProvider *dpp;
 464     CephContext *cct;
 465     RGWLC *lc;
 466     Mutex lock;
 467     Cond cond;
 468
 469   public:
 470     LCWorker(const DoutPrefixProvider* _dpp, CephContext *_cct, RGWLC *_lc) : dpp(_dpp), cct(_cct), lc(_lc), lock("LCWorker") {}
 471     void *entry() override;
 472     void stop();
 473     bool should_work(utime_t& now);
 474     int schedule_next_start_time(utime_t& start, utime_t& now);
 475   };
 476
 477   public:
 478   LCWorker *worker;
 479   RGWLC() : cct(NULL), store(NULL), worker(NULL) {}
 480   ~RGWLC() {
 481     stop_processor();
 482     finalize();
 483   }
 484
 485   void initialize(CephContext *_cct, RGWRados *_store);
 486   void finalize();
 487
 488   int process();
 489   int process(int index, int max_secs);
 490   bool if_already_run_today(time_t& start_date);
 491   int list_lc_progress(const string& marker, uint32_t max_entries, map<string, int> *progress_map);
 492   int bucket_lc_prepare(int index);
 493   int bucket_lc_process(string& shard_id);
 494   int bucket_lc_post(int index, int max_lock_sec, pair<string, int >& entry, int& result);
 495   bool going_down();
 496   void start_processor();
 497   void stop_processor();
 498   int set_bucket_config(RGWBucketInfo& bucket_info,
 499                         const map<string, bufferlist>& bucket_attrs,
 500                         RGWLifecycleConfiguration *config);
 501   int remove_bucket_config(RGWBucketInfo& bucket_info,
 502                            const map<string, bufferlist>& bucket_attrs);
 503
 504   CephContext *get_cct() const override { return store->ctx(); }
 505   unsigned get_subsys() const;
 506   std::ostream& gen_prefix(std::ostream& out) const;
 507
 508   private:
 509
 510   int handle_multipart_expiration(RGWRados::Bucket *target,
 511                                   const multimap<string, lc_op>& prefix_map);
 512 };
 513
 514 namespace rgw::lc {
 515
 516 int fix_lc_shard_entry(RGWRados *store, const RGWBucketInfo& bucket_info,
 517                        const map<std::string,bufferlist>& battrs);
 518
 519 } // namespace rgw::lc
 520
 521 #endif