ceph/src/rgw/rgw_lc_s3.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab ft=cpp
   3
   4 #include <string.h>
   5
   6 #include <iostream>
   7 #include <map>
   8
   9 #include "include/types.h"
  10
  11 #include "rgw_user.h"
  12 #include "rgw_lc_s3.h"
  13
  14
  15 #define dout_subsys ceph_subsys_rgw
  16
  17 using namespace std;
  18
  19 static bool check_date(const string& _date)
  20 {
  21   boost::optional<ceph::real_time> date = ceph::from_iso_8601(_date);
  22   if (boost::none == date) {
  23     return false;
  24   }
  25   struct timespec time = ceph::real_clock::to_timespec(*date);
  26   if (time.tv_sec % (24*60*60) || time.tv_nsec) {
  27     return false;
  28   }
  29   return true;
  30 }
  31
  32 void LCExpiration_S3::dump_xml(Formatter *f) const {
  33   if (dm_expiration) {
  34     encode_xml("ExpiredObjectDeleteMarker", "true", f);
  35   } else if (!days.empty()) {
  36     encode_xml("Days", days, f);
  37   } else {
  38     encode_xml("Date", date, f);
  39   }
  40 }
  41
  42 void LCExpiration_S3::decode_xml(XMLObj *obj)
  43 {
  44   bool has_days = RGWXMLDecoder::decode_xml("Days", days, obj);
  45   bool has_date = RGWXMLDecoder::decode_xml("Date", date, obj);
  46   string dm;
  47   bool has_dm = RGWXMLDecoder::decode_xml("ExpiredObjectDeleteMarker", dm, obj);
  48
  49   int num = !!has_days + !!has_date + !!has_dm;
  50
  51   if (num != 1) {
  52     throw RGWXMLDecoder::err("bad Expiration section");
  53   }
  54
  55   if (has_date && !check_date(date)) {
  56     //We need return xml error according to S3
  57     throw RGWXMLDecoder::err("bad date in Date section");
  58   }
  59
  60   if (has_dm) {
  61     dm_expiration = (dm == "true");
  62   }
  63 }
  64
  65 void LCNoncurExpiration_S3::decode_xml(XMLObj *obj)
  66 {
  67   RGWXMLDecoder::decode_xml("NoncurrentDays", days, obj, true);
  68 }
  69
  70 void LCNoncurExpiration_S3::dump_xml(Formatter *f) const
  71 {
  72   encode_xml("NoncurrentDays", days, f);
  73 }
  74
  75 void LCMPExpiration_S3::decode_xml(XMLObj *obj)
  76 {
  77   RGWXMLDecoder::decode_xml("DaysAfterInitiation", days, obj, true);
  78 }
  79
  80 void LCMPExpiration_S3::dump_xml(Formatter *f) const
  81 {
  82   encode_xml("DaysAfterInitiation", days, f);
  83 }
  84
  85 void RGWLifecycleConfiguration_S3::decode_xml(XMLObj *obj)
  86 {
  87   if (!cct) {
  88     throw RGWXMLDecoder::err("ERROR: RGWLifecycleConfiguration_S3 can't be decoded without cct initialized");
  89   }
  90   vector<LCRule_S3> rules;
  91
  92   RGWXMLDecoder::decode_xml("Rule", rules, obj, true);
  93
  94   for (auto& rule : rules) {
  95     if (rule.get_id().empty()) {
  96       // S3 generates a 48 bit random ID, maybe we could generate shorter IDs
  97       static constexpr auto LC_ID_LENGTH = 48;
  98       string id = gen_rand_alphanumeric_lower(cct, LC_ID_LENGTH);
  99       rule.set_id(id);
 100     }
 101
 102     add_rule(rule);
 103   }
 104
 105   if (cct->_conf->rgw_lc_max_rules < rule_map.size()) {
 106     stringstream ss;
 107     ss << "Warn: The lifecycle config has too many rules, rule number is:"
 108       << rule_map.size() << ", max number is:" << cct->_conf->rgw_lc_max_rules;
 109     throw RGWXMLDecoder::err(ss.str());
 110   }
 111 }
 112
 113 void LCFilter_S3::dump_xml(Formatter *f) const
 114 {
 115   if (has_prefix()) {
 116     encode_xml("Prefix", prefix, f);
 117   }
 118   bool multi = has_multi_condition();
 119   if (multi) {
 120     f->open_array_section("And");
 121   }
 122   if (has_tags()) {
 123     const auto& tagset_s3 = static_cast<const RGWObjTagSet_S3 &>(obj_tags);
 124     tagset_s3.dump_xml(f);
 125   }
 126   if (multi) {
 127     f->close_section();
 128   }
 129 }
 130
 131 void LCFilter_S3::decode_xml(XMLObj *obj)
 132 {
 133   /*
 134    * The prior logic here looked for an And element, but did not
 135    * structurally parse the Filter clause (and incorrectly rejected
 136    * the base case where a Prefix and one Tag were supplied).  It
 137    * could not reject generally malformed Filter syntax.
 138    *
 139    * Empty filters are allowed:
 140    * https://docs.aws.amazon.com/AmazonS3/latest/dev/intro-lifecycle-rules.html
 141    */
 142   XMLObj *o = obj->find_first("And");
 143   if (o == nullptr){
 144     o = obj;
 145   }
 146
 147   RGWXMLDecoder::decode_xml("Prefix", prefix, o);
 148   auto tags_iter = o->find("Tag");
 149   obj_tags.clear();
 150   while (auto tag_xml =tags_iter.get_next()){
 151     std::string _key,_val;
 152     RGWXMLDecoder::decode_xml("Key", _key, tag_xml);
 153     RGWXMLDecoder::decode_xml("Value", _val, tag_xml);
 154     obj_tags.emplace_tag(std::move(_key), std::move(_val));
 155   }
 156 }
 157
 158 void LCTransition_S3::decode_xml(XMLObj *obj)
 159 {
 160   bool has_days = RGWXMLDecoder::decode_xml("Days", days, obj);
 161   bool has_date = RGWXMLDecoder::decode_xml("Date", date, obj);
 162   if ((has_days && has_date) || (!has_days && !has_date)) {
 163     throw RGWXMLDecoder::err("bad Transition section");
 164   }
 165
 166   if (has_date && !check_date(date)) {
 167     //We need return xml error according to S3
 168     throw RGWXMLDecoder::err("bad Date in Transition section");
 169   }
 170
 171   if (!RGWXMLDecoder::decode_xml("StorageClass", storage_class, obj)) {
 172     throw RGWXMLDecoder::err("missing StorageClass in Transition section");
 173   }
 174 }
 175
 176 void LCTransition_S3::dump_xml(Formatter *f) const {
 177   if (!days.empty()) {
 178     encode_xml("Days", days, f);
 179   } else {
 180     encode_xml("Date", date, f);
 181   }
 182   encode_xml("StorageClass", storage_class, f);
 183 }
 184
 185 void LCNoncurTransition_S3::decode_xml(XMLObj *obj)
 186 {
 187   if (!RGWXMLDecoder::decode_xml("NoncurrentDays", days, obj)) {
 188     throw RGWXMLDecoder::err("missing NoncurrentDays in NoncurrentVersionTransition section");
 189   }
 190   if (!RGWXMLDecoder::decode_xml("StorageClass", storage_class, obj)) {
 191     throw RGWXMLDecoder::err("missing StorageClass in NoncurrentVersionTransition section");
 192   }
 193 }
 194
 195 void LCNoncurTransition_S3::dump_xml(Formatter *f) const
 196 {
 197   encode_xml("NoncurrentDays", days, f);
 198   encode_xml("StorageClass", storage_class, f);
 199 }
 200
 201 void LCRule_S3::decode_xml(XMLObj *obj)
 202 {
 203   id.clear();
 204   prefix.clear();
 205   status.clear();
 206   dm_expiration = false;
 207
 208   RGWXMLDecoder::decode_xml("ID", id, obj);
 209
 210   LCFilter_S3 filter_s3;
 211   if (!RGWXMLDecoder::decode_xml("Filter", filter_s3, obj)) {
 212     // Ideally the following code should be deprecated and we should return
 213     // False here, The new S3 LC configuration xml spec. makes Filter mandatory
 214     // and Prefix optional. However older clients including boto2 still generate
 215     // xml according to the older spec, where Prefix existed outside of Filter
 216     // and S3 itself seems to be sloppy on enforcing the mandatory Filter
 217     // argument. A day will come when S3 enforces their own xml-spec, but it is
 218     // not this day
 219
 220     if (!RGWXMLDecoder::decode_xml("Prefix", prefix, obj)) {
 221       throw RGWXMLDecoder::err("missing Prefix in Filter");
 222     }
 223   }
 224   filter = (LCFilter)filter_s3;
 225
 226   if (!RGWXMLDecoder::decode_xml("Status", status, obj)) {
 227     throw RGWXMLDecoder::err("missing Status in Filter");
 228   }
 229   if (status.compare("Enabled") != 0 && status.compare("Disabled") != 0) {
 230     throw RGWXMLDecoder::err("bad Status in Filter");
 231   }
 232
 233   LCExpiration_S3 s3_expiration;
 234   LCNoncurExpiration_S3 s3_noncur_expiration;
 235   LCMPExpiration_S3 s3_mp_expiration;
 236   LCFilter_S3 s3_filter;
 237
 238   bool has_expiration = RGWXMLDecoder::decode_xml("Expiration", s3_expiration, obj);
 239   bool has_noncur_expiration = RGWXMLDecoder::decode_xml("NoncurrentVersionExpiration", s3_noncur_expiration, obj);
 240   bool has_mp_expiration = RGWXMLDecoder::decode_xml("AbortIncompleteMultipartUpload", s3_mp_expiration, obj);
 241
 242   vector<LCTransition_S3> transitions;
 243   vector<LCNoncurTransition_S3> noncur_transitions;
 244
 245   bool has_transition = RGWXMLDecoder::decode_xml("Transition", transitions, obj);
 246   bool has_noncur_transition = RGWXMLDecoder::decode_xml("NoncurrentVersionTransition", noncur_transitions, obj);
 247
 248   if (!has_expiration &&
 249       !has_noncur_expiration &&
 250       !has_mp_expiration &&
 251       !has_transition &&
 252       !has_noncur_transition) {
 253     throw RGWXMLDecoder::err("bad Rule");
 254   }
 255
 256   if (has_expiration) {
 257     if (s3_expiration.has_days() ||
 258         s3_expiration.has_date()) {
 259       expiration = s3_expiration;
 260     } else {
 261       dm_expiration = s3_expiration.get_dm_expiration();
 262     }
 263   }
 264   if (has_noncur_expiration) {
 265     noncur_expiration = s3_noncur_expiration;
 266   }
 267   if (has_mp_expiration) {
 268     mp_expiration = s3_mp_expiration;
 269   }
 270   for (auto& t : transitions) {
 271     if (!add_transition(t)) {
 272       throw RGWXMLDecoder::err("Failed to add transition");
 273     }
 274   }
 275   for (auto& t : noncur_transitions) {
 276     if (!add_noncur_transition(t)) {
 277       throw RGWXMLDecoder::err("Failed to add non-current version transition");
 278     }
 279   }
 280 }
 281
 282 void LCRule_S3::dump_xml(Formatter *f) const {
 283   encode_xml("ID", id, f);
 284   // In case of an empty filter and an empty Prefix, we defer to Prefix.
 285   if (!filter.empty()) {
 286     const LCFilter_S3& lc_filter = static_cast<const LCFilter_S3&>(filter);
 287     encode_xml("Filter", lc_filter, f);
 288   } else {
 289     encode_xml("Prefix", prefix, f);
 290   }
 291   encode_xml("Status", status, f);
 292   if (!expiration.empty() || dm_expiration) {
 293     LCExpiration_S3 expir(expiration.get_days_str(), expiration.get_date(), dm_expiration);
 294     encode_xml("Expiration", expir, f);
 295   }
 296   if (!noncur_expiration.empty()) {
 297     const LCNoncurExpiration_S3& noncur_expir = static_cast<const LCNoncurExpiration_S3&>(noncur_expiration);
 298     encode_xml("NoncurrentVersionExpiration", noncur_expir, f);
 299   }
 300   if (!mp_expiration.empty()) {
 301     const LCMPExpiration_S3& mp_expir = static_cast<const LCMPExpiration_S3&>(mp_expiration);
 302     encode_xml("AbortIncompleteMultipartUpload", mp_expir, f);
 303   }
 304   if (!transitions.empty()) {
 305     for (auto &elem : transitions) {
 306       const LCTransition_S3& tran = static_cast<const LCTransition_S3&>(elem.second);
 307       encode_xml("Transition", tran, f);
 308     }
 309   }
 310   if (!noncur_transitions.empty()) {
 311     for (auto &elem : noncur_transitions) {
 312       const LCNoncurTransition_S3& noncur_tran = static_cast<const LCNoncurTransition_S3&>(elem.second);
 313       encode_xml("NoncurrentVersionTransition", noncur_tran, f);
 314     }
 315   }
 316 }
 317
 318 int RGWLifecycleConfiguration_S3::rebuild(RGWLifecycleConfiguration& dest)
 319 {
 320   int ret = 0;
 321   multimap<string, LCRule>::iterator iter;
 322   for (iter = rule_map.begin(); iter != rule_map.end(); ++iter) {
 323     LCRule& src_rule = iter->second;
 324     ret = dest.check_and_add_rule(src_rule);
 325     if (ret < 0)
 326       return ret;
 327   }
 328   if (!dest.valid()) {
 329     ret = -ERR_INVALID_REQUEST;
 330   }
 331   return ret;
 332 }
 333
 334
 335 void RGWLifecycleConfiguration_S3::dump_xml(Formatter *f) const
 336 {
 337   for (auto iter = rule_map.begin(); iter != rule_map.end(); ++iter) {
 338     const LCRule_S3& rule = static_cast<const LCRule_S3&>(iter->second);
 339     encode_xml("Rule", rule, f);
 340   }
 341 }
 342