ceph/src/rgw/rgw_lc.h

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab ft=cpp
   3
   4 #ifndef CEPH_RGW_LC_H
   5 #define CEPH_RGW_LC_H
   6
   7 #include <map>
   8 #include <string>
   9 #include <iostream>
  10
  11 #include "common/debug.h"
  12
  13 #include "include/types.h"
  14 #include "include/rados/librados.hpp"
  15 #include "common/ceph_mutex.h"
  16 #include "common/Cond.h"
  17 #include "common/iso_8601.h"
  18 #include "common/Thread.h"
  19 #include "rgw_common.h"
  20 #include "cls/rgw/cls_rgw_types.h"
  21 #include "rgw_tag.h"
  22 #include "rgw_sal.h"
  23 #include "rgw_rados.h"
  24
  25 #include <atomic>
  26 #include <tuple>
  27
  28 #define HASH_PRIME 7877
  29 #define MAX_ID_LEN 255
  30 static string lc_oid_prefix = "lc";
  31 static string lc_index_lock_name = "lc_process";
  32
  33 extern const char* LC_STATUS[];
  34
  35 typedef enum {
  36   lc_uninitial = 0,
  37   lc_processing,
  38   lc_failed,
  39   lc_complete,
  40 } LC_BUCKET_STATUS;
  41
  42 class LCExpiration
  43 {
  44 protected:
  45   string days;
  46   //At present only current object has expiration date
  47   string date;
  48 public:
  49   LCExpiration() {}
  50   LCExpiration(const string& _days, const string& _date) : days(_days), date(_date) {}
  51
  52   void encode(bufferlist& bl) const {
  53     ENCODE_START(3, 2, bl);
  54     encode(days, bl);
  55     encode(date, bl);
  56     ENCODE_FINISH(bl);
  57   }
  58   void decode(bufferlist::const_iterator& bl) {
  59     DECODE_START_LEGACY_COMPAT_LEN(3, 2, 2, bl);
  60     decode(days, bl);
  61     if (struct_v >= 3) {
  62       decode(date, bl);
  63     }
  64     DECODE_FINISH(bl);
  65   }
  66   void dump(Formatter *f) const;
  67 //  static void generate_test_instances(list<ACLOwner*>& o);
  68   void set_days(const string& _days) { days = _days; }
  69   string get_days_str() const {
  70     return days;
  71   }
  72   int get_days() const {return atoi(days.c_str()); }
  73   bool has_days() const {
  74     return !days.empty();
  75   }
  76   void set_date(const string& _date) { date = _date; }
  77   string get_date() const {
  78     return date;
  79   }
  80   bool has_date() const {
  81     return !date.empty();
  82   }
  83   bool empty() const {
  84     return days.empty() && date.empty();
  85   }
  86   bool valid() const {
  87     if (!days.empty() && !date.empty()) {
  88       return false;
  89     } else if (!days.empty() && get_days() <= 0) {
  90       return false;
  91     }
  92     //We've checked date in xml parsing
  93     return true;
  94   }
  95 };
  96 WRITE_CLASS_ENCODER(LCExpiration)
  97
  98 class LCTransition
  99 {
 100 protected:
 101   string days;
 102   string date;
 103   string storage_class;
 104
 105 public:
 106   int get_days() const {
 107     return atoi(days.c_str());
 108   }
 109
 110   string get_date() const {
 111     return date;
 112   }
 113
 114   string get_storage_class() const {
 115     return storage_class;
 116   }
 117
 118   bool has_days() const {
 119     return !days.empty();
 120   }
 121
 122   bool has_date() const {
 123     return !date.empty();
 124   }
 125
 126   bool empty() const {
 127     return days.empty() && date.empty();
 128   }
 129
 130   bool valid() const {
 131     if (!days.empty() && !date.empty()) {
 132       return false;
 133     } else if (!days.empty() && get_days() < 0) {
 134       return false;
 135     }
 136     //We've checked date in xml parsing
 137     return true;
 138   }
 139
 140   void encode(bufferlist& bl) const {
 141     ENCODE_START(1, 1, bl);
 142     encode(days, bl);
 143     encode(date, bl);
 144     encode(storage_class, bl);
 145     ENCODE_FINISH(bl);
 146   }
 147
 148   void decode(bufferlist::const_iterator& bl) {
 149     DECODE_START(1, bl);
 150     decode(days, bl);
 151     decode(date, bl);
 152     decode(storage_class, bl);
 153     DECODE_FINISH(bl);
 154   }
 155   void dump(Formatter *f) const {
 156     f->dump_string("days", days);
 157     f->dump_string("date", date);
 158     f->dump_string("storage_class", storage_class);
 159   }
 160 };
 161 WRITE_CLASS_ENCODER(LCTransition)
 162
 163 class LCFilter
 164 {
 165  protected:
 166   std::string prefix;
 167   RGWObjTags obj_tags;
 168
 169  public:
 170
 171   const std::string& get_prefix() const {
 172     return prefix;
 173   }
 174
 175   const RGWObjTags& get_tags() const {
 176     return obj_tags;
 177   }
 178
 179   bool empty() const {
 180     return !(has_prefix() || has_tags());
 181   }
 182
 183   // Determine if we need AND tag when creating xml
 184   bool has_multi_condition() const {
 185     if (obj_tags.count() > 1)
 186       return true;
 187     return false;
 188   }
 189
 190   bool has_prefix() const {
 191     return !prefix.empty();
 192   }
 193
 194   bool has_tags() const {
 195     return !obj_tags.empty();
 196   }
 197
 198   void encode(bufferlist& bl) const {
 199     ENCODE_START(2, 1, bl);
 200     encode(prefix, bl);
 201     encode(obj_tags, bl);
 202     ENCODE_FINISH(bl);
 203   }
 204   void decode(bufferlist::const_iterator& bl) {
 205     DECODE_START(2, bl);
 206     decode(prefix, bl);
 207     if (struct_v >= 2) {
 208       decode(obj_tags, bl);
 209     }
 210     DECODE_FINISH(bl);
 211   }
 212   void dump(Formatter *f) const;
 213 };
 214 WRITE_CLASS_ENCODER(LCFilter)
 215
 216 class LCRule
 217 {
 218 protected:
 219   string id;
 220   string prefix;
 221   string status;
 222   LCExpiration expiration;
 223   LCExpiration noncur_expiration;
 224   LCExpiration mp_expiration;
 225   LCFilter filter;
 226   map<string, LCTransition> transitions;
 227   map<string, LCTransition> noncur_transitions;
 228   bool dm_expiration = false;
 229
 230 public:
 231
 232   LCRule(){};
 233   ~LCRule(){};
 234
 235   const string& get_id() const {
 236       return id;
 237   }
 238
 239   const string& get_status() const {
 240       return status;
 241   }
 242
 243   bool is_enabled() const {
 244     return status == "Enabled";
 245   }
 246
 247   void set_enabled(bool flag) {
 248     status = (flag ? "Enabled" : "Disabled");
 249   }
 250
 251   const string& get_prefix() const {
 252       return prefix;
 253   }
 254
 255   const LCFilter& get_filter() const {
 256     return filter;
 257   }
 258
 259   const LCExpiration& get_expiration() const {
 260     return expiration;
 261   }
 262
 263   const LCExpiration& get_noncur_expiration() const {
 264     return noncur_expiration;
 265   }
 266
 267   const LCExpiration& get_mp_expiration() const {
 268     return mp_expiration;
 269   }
 270
 271   bool get_dm_expiration() const {
 272     return dm_expiration;
 273   }
 274
 275   const map<string, LCTransition>& get_transitions() const {
 276     return transitions;
 277   }
 278
 279   const map<string, LCTransition>& get_noncur_transitions() const {
 280     return noncur_transitions;
 281   }
 282
 283   void set_id(const string& _id) {
 284     id = _id;
 285   }
 286
 287   void set_prefix(const string& _prefix) {
 288     prefix = _prefix;
 289   }
 290
 291   void set_status(const string& _status) {
 292     status = _status;
 293   }
 294
 295   void set_expiration(const LCExpiration& _expiration) {
 296     expiration = _expiration;
 297   }
 298
 299   void set_noncur_expiration(const LCExpiration& _noncur_expiration) {
 300     noncur_expiration = _noncur_expiration;
 301   }
 302
 303   void set_mp_expiration(const LCExpiration& _mp_expiration) {
 304     mp_expiration = _mp_expiration;
 305   }
 306
 307   void set_dm_expiration(bool _dm_expiration) {
 308     dm_expiration = _dm_expiration;
 309   }
 310
 311   bool add_transition(const LCTransition& _transition) {
 312     auto ret = transitions.emplace(_transition.get_storage_class(), _transition);
 313     return ret.second;
 314   }
 315
 316   bool add_noncur_transition(const LCTransition& _noncur_transition) {
 317     auto ret = noncur_transitions.emplace(_noncur_transition.get_storage_class(), _noncur_transition);
 318     return ret.second;
 319   }
 320
 321   bool valid() const;
 322
 323   void encode(bufferlist& bl) const {
 324      ENCODE_START(6, 1, bl);
 325      encode(id, bl);
 326      encode(prefix, bl);
 327      encode(status, bl);
 328      encode(expiration, bl);
 329      encode(noncur_expiration, bl);
 330      encode(mp_expiration, bl);
 331      encode(dm_expiration, bl);
 332      encode(filter, bl);
 333      encode(transitions, bl);
 334      encode(noncur_transitions, bl);
 335      ENCODE_FINISH(bl);
 336    }
 337    void decode(bufferlist::const_iterator& bl) {
 338      DECODE_START_LEGACY_COMPAT_LEN(6, 1, 1, bl);
 339      decode(id, bl);
 340      decode(prefix, bl);
 341      decode(status, bl);
 342      decode(expiration, bl);
 343      if (struct_v >=2) {
 344        decode(noncur_expiration, bl);
 345      }
 346      if (struct_v >= 3) {
 347        decode(mp_expiration, bl);
 348      }
 349      if (struct_v >= 4) {
 350         decode(dm_expiration, bl);
 351      }
 352      if (struct_v >= 5) {
 353        decode(filter, bl);
 354      }
 355      if (struct_v >= 6) {
 356        decode(transitions, bl);
 357        decode(noncur_transitions, bl);
 358      }
 359      DECODE_FINISH(bl);
 360    }
 361   void dump(Formatter *f) const;
 362
 363   void init_simple_days_rule(std::string_view _id, std::string_view _prefix, int num_days);
 364 };
 365 WRITE_CLASS_ENCODER(LCRule)
 366
 367 struct transition_action
 368 {
 369   int days;
 370   boost::optional<ceph::real_time> date;
 371   string storage_class;
 372   transition_action() : days(0) {}
 373   void dump(Formatter *f) const {
 374     if (!date) {
 375       f->dump_int("days", days);
 376     } else {
 377       utime_t ut(*date);
 378       f->dump_stream("date") << ut;
 379     }
 380   }
 381 };
 382
 383 /* XXX why not LCRule? */
 384 struct lc_op
 385 {
 386   string id;
 387   bool status{false};
 388   bool dm_expiration{false};
 389   int expiration{0};
 390   int noncur_expiration{0};
 391   int mp_expiration{0};
 392   boost::optional<ceph::real_time> expiration_date;
 393   boost::optional<RGWObjTags> obj_tags;
 394   map<string, transition_action> transitions;
 395   map<string, transition_action> noncur_transitions;
 396
 397   /* ctors are nice */
 398   lc_op() = delete;
 399
 400   lc_op(const std::string id) : id(id)
 401     {}
 402
 403   void dump(Formatter *f) const;
 404 };
 405
 406 class RGWLifecycleConfiguration
 407 {
 408 protected:
 409   CephContext *cct;
 410   multimap<string, lc_op> prefix_map;
 411   multimap<string, LCRule> rule_map;
 412   bool _add_rule(const LCRule& rule);
 413   bool has_same_action(const lc_op& first, const lc_op& second);
 414 public:
 415   explicit RGWLifecycleConfiguration(CephContext *_cct) : cct(_cct) {}
 416   RGWLifecycleConfiguration() : cct(NULL) {}
 417
 418   void set_ctx(CephContext *ctx) {
 419     cct = ctx;
 420   }
 421
 422   virtual ~RGWLifecycleConfiguration() {}
 423
 424 //  int get_perm(string& id, int perm_mask);
 425 //  int get_group_perm(ACLGroupTypeEnum group, int perm_mask);
 426   void encode(bufferlist& bl) const {
 427     ENCODE_START(1, 1, bl);
 428     encode(rule_map, bl);
 429     ENCODE_FINISH(bl);
 430   }
 431   void decode(bufferlist::const_iterator& bl) {
 432     DECODE_START_LEGACY_COMPAT_LEN(1, 1, 1, bl);
 433     decode(rule_map, bl);
 434     multimap<string, LCRule>::iterator iter;
 435     for (iter = rule_map.begin(); iter != rule_map.end(); ++iter) {
 436       LCRule& rule = iter->second;
 437       _add_rule(rule);
 438     }
 439     DECODE_FINISH(bl);
 440   }
 441   void dump(Formatter *f) const;
 442   static void generate_test_instances(list<RGWLifecycleConfiguration*>& o);
 443
 444   void add_rule(const LCRule& rule);
 445
 446   int check_and_add_rule(const LCRule& rule);
 447
 448   bool valid();
 449
 450   multimap<string, LCRule>& get_rule_map() { return rule_map; }
 451   multimap<string, lc_op>& get_prefix_map() { return prefix_map; }
 452 /*
 453   void create_default(string id, string name) {
 454     ACLGrant grant;
 455     grant.set_canon(id, name, RGW_PERM_FULL_CONTROL);
 456     add_grant(&grant);
 457   }
 458 */
 459 };
 460 WRITE_CLASS_ENCODER(RGWLifecycleConfiguration)
 461
 462 class RGWLC : public DoutPrefixProvider {
 463   CephContext *cct;
 464   rgw::sal::RGWRadosStore *store;
 465   std::unique_ptr<rgw::sal::Lifecycle> sal_lc;
 466   int max_objs{0};
 467   string *obj_names{nullptr};
 468   std::atomic<bool> down_flag = { false };
 469   string cookie;
 470
 471 public:
 472
 473   class WorkPool;
 474
 475   class LCWorker : public Thread
 476   {
 477     const DoutPrefixProvider *dpp;
 478     CephContext *cct;
 479     RGWLC *lc;
 480     int ix;
 481     std::mutex lock;
 482     std::condition_variable cond;
 483     WorkPool* workpool{nullptr};
 484
 485   public:
 486
 487     using lock_guard = std::lock_guard<std::mutex>;
 488     using unique_lock = std::unique_lock<std::mutex>;
 489
 490     LCWorker(const DoutPrefixProvider* dpp, CephContext *_cct, RGWLC *_lc,
 491              int ix);
 492     RGWLC* get_lc() { return lc; }
 493     void *entry() override;
 494     void stop();
 495     bool should_work(utime_t& now);
 496     int schedule_next_start_time(utime_t& start, utime_t& now);
 497     ~LCWorker();
 498
 499     friend class RGWRados;
 500     friend class RGWLC;
 501     friend class WorkQ;
 502   }; /* LCWorker */
 503
 504   friend class RGWRados;
 505
 506   std::vector<std::unique_ptr<RGWLC::LCWorker>> workers;
 507
 508   RGWLC() : cct(nullptr), store(nullptr) {}
 509   ~RGWLC();
 510
 511   void initialize(CephContext *_cct, rgw::sal::RGWRadosStore *_store);
 512   void finalize();
 513
 514   int process(LCWorker* worker, bool once);
 515   int process(int index, int max_secs, LCWorker* worker, bool once);
 516   bool if_already_run_today(time_t start_date);
 517   bool expired_session(time_t started);
 518   time_t thread_stop_at();
 519   int list_lc_progress(string& marker, uint32_t max_entries,
 520                        vector<rgw::sal::Lifecycle::LCEntry>&, int& index);
 521   int bucket_lc_prepare(int index, LCWorker* worker);
 522   int bucket_lc_process(string& shard_id, LCWorker* worker, time_t stop_at,
 523                         bool once);
 524   int bucket_lc_post(int index, int max_lock_sec,
 525                      rgw::sal::Lifecycle::LCEntry& entry, int& result, LCWorker* worker);
 526   bool going_down();
 527   void start_processor();
 528   void stop_processor();
 529   int set_bucket_config(RGWBucketInfo& bucket_info,
 530                         const map<string, bufferlist>& bucket_attrs,
 531                         RGWLifecycleConfiguration *config);
 532   int remove_bucket_config(RGWBucketInfo& bucket_info,
 533                            const map<string, bufferlist>& bucket_attrs);
 534
 535   CephContext *get_cct() const override { return cct; }
 536   rgw::sal::Lifecycle *get_lc() const { return sal_lc.get(); }
 537   unsigned get_subsys() const;
 538   std::ostream& gen_prefix(std::ostream& out) const;
 539
 540   private:
 541
 542   int handle_multipart_expiration(rgw::sal::RGWBucket* target,
 543                                   const multimap<string, lc_op>& prefix_map,
 544                                   LCWorker* worker, time_t stop_at, bool once);
 545 };
 546
 547 namespace rgw::lc {
 548
 549 int fix_lc_shard_entry(rgw::sal::RGWRadosStore *store,
 550                        rgw::sal::Lifecycle* sal_lc,
 551                        const RGWBucketInfo& bucket_info,
 552                        const map<std::string,bufferlist>& battrs);
 553
 554 std::string s3_expiration_header(
 555   DoutPrefixProvider* dpp,
 556   const rgw_obj_key& obj_key,
 557   const RGWObjTags& obj_tagset,
 558   const ceph::real_time& mtime,
 559   const std::map<std::string, buffer::list>& bucket_attrs);
 560
 561 bool s3_multipart_abort_header(
 562   DoutPrefixProvider* dpp,
 563   const rgw_obj_key& obj_key,
 564   const ceph::real_time& mtime,
 565   const std::map<std::string, buffer::list>& bucket_attrs,
 566   ceph::real_time& abort_date,
 567   std::string& rule_id);
 568
 569 } // namespace rgw::lc
 570
 571 #endif