ceph/src/rgw/rgw_lc_s3.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab ft=cpp
   3
   4 #include <string.h>
   5
   6 #include <iostream>
   7 #include <map>
   8
   9 #include "include/types.h"
  10
  11 #include "rgw_user.h"
  12 #include "rgw_lc_s3.h"
  13
  14
  15 #define dout_subsys ceph_subsys_rgw
  16
  17 using namespace std;
  18
  19 static bool check_date(const string& _date)
  20 {
  21   boost::optional<ceph::real_time> date = ceph::from_iso_8601(_date);
  22   if (boost::none == date) {
  23     return false;
  24   }
  25   struct timespec time = ceph::real_clock::to_timespec(*date);
  26   if (time.tv_sec % (24*60*60) || time.tv_nsec) {
  27     return false;
  28   }
  29   return true;
  30 }
  31
  32 void LCExpiration_S3::dump_xml(Formatter *f) const {
  33   if (dm_expiration) {
  34     encode_xml("ExpiredObjectDeleteMarker", "true", f);
  35   } else if (!days.empty()) {
  36     encode_xml("Days", days, f);
  37   } else {
  38     encode_xml("Date", date, f);
  39   }
  40 }
  41
  42 void LCExpiration_S3::decode_xml(XMLObj *obj)
  43 {
  44   bool has_days = RGWXMLDecoder::decode_xml("Days", days, obj);
  45   bool has_date = RGWXMLDecoder::decode_xml("Date", date, obj);
  46   string dm;
  47   bool has_dm = RGWXMLDecoder::decode_xml("ExpiredObjectDeleteMarker", dm, obj);
  48
  49   int num = !!has_days + !!has_date + !!has_dm;
  50
  51   if (num != 1) {
  52     throw RGWXMLDecoder::err("bad Expiration section");
  53   }
  54
  55   if (has_date && !check_date(date)) {
  56     //We need return xml error according to S3
  57     throw RGWXMLDecoder::err("bad date in Date section");
  58   }
  59
  60   if (has_dm) {
  61     dm_expiration = (dm == "true");
  62   }
  63 }
  64
  65 void LCNoncurExpiration_S3::decode_xml(XMLObj *obj)
  66 {
  67   RGWXMLDecoder::decode_xml("NoncurrentDays", days, obj, true);
  68 }
  69
  70 void LCNoncurExpiration_S3::dump_xml(Formatter *f) const
  71 {
  72   encode_xml("NoncurrentDays", days, f);
  73 }
  74
  75 void LCMPExpiration_S3::decode_xml(XMLObj *obj)
  76 {
  77   RGWXMLDecoder::decode_xml("DaysAfterInitiation", days, obj, true);
  78 }
  79
  80 void LCMPExpiration_S3::dump_xml(Formatter *f) const
  81 {
  82   encode_xml("DaysAfterInitiation", days, f);
  83 }
  84
  85 void RGWLifecycleConfiguration_S3::decode_xml(XMLObj *obj)
  86 {
  87   if (!cct) {
  88     throw RGWXMLDecoder::err("ERROR: RGWLifecycleConfiguration_S3 can't be decoded without cct initialized");
  89   }
  90   vector<LCRule_S3> rules;
  91
  92   RGWXMLDecoder::decode_xml("Rule", rules, obj, true);
  93
  94   for (auto& rule : rules) {
  95     if (rule.get_id().empty()) {
  96       // S3 generates a 48 bit random ID, maybe we could generate shorter IDs
  97       static constexpr auto LC_ID_LENGTH = 48;
  98       string id = gen_rand_alphanumeric_lower(cct, LC_ID_LENGTH);
  99       rule.set_id(id);
 100     }
 101
 102     add_rule(rule);
 103   }
 104
 105   if (cct->_conf->rgw_lc_max_rules < rule_map.size()) {
 106     stringstream ss;
 107     ss << "Warn: The lifecycle config has too many rules, rule number is:"
 108       << rule_map.size() << ", max number is:" << cct->_conf->rgw_lc_max_rules;
 109     throw RGWXMLDecoder::err(ss.str());
 110   }
 111 }
 112
 113 void LCFilter_S3::dump_xml(Formatter *f) const
 114 {
 115   bool multi = has_multi_condition();
 116   if (multi) {
 117     f->open_array_section("And");
 118   }
 119   if (has_prefix()) {
 120     encode_xml("Prefix", prefix, f);
 121   }
 122   if (has_tags()) {
 123     const auto& tagset_s3 = static_cast<const RGWObjTagSet_S3 &>(obj_tags);
 124     tagset_s3.dump_xml(f);
 125   }
 126   if (has_flags()) {
 127     if (have_flag(LCFlagType::ArchiveZone)) {
 128       encode_xml("ArchiveZone", "", f);
 129     }
 130   }
 131   if (multi) {
 132     f->close_section(); // And
 133   }
 134 }
 135
 136 void LCFilter_S3::decode_xml(XMLObj *obj)
 137 {
 138   /*
 139    * The prior logic here looked for an And element, but did not
 140    * structurally parse the Filter clause (and incorrectly rejected
 141    * the base case where a Prefix and one Tag were supplied).  It
 142    * could not reject generally malformed Filter syntax.
 143    *
 144    * Empty filters are allowed:
 145    * https://docs.aws.amazon.com/AmazonS3/latest/dev/intro-lifecycle-rules.html
 146    */
 147   XMLObj* o = obj->find_first("And");
 148   if (o == nullptr){
 149     o = obj;
 150   }
 151
 152   RGWXMLDecoder::decode_xml("Prefix", prefix, o);
 153
 154   /* parse optional ArchiveZone flag (extension) */
 155   if (o->find_first("ArchiveZone")) {
 156     flags |= make_flag(LCFlagType::ArchiveZone);
 157   }
 158
 159   obj_tags.clear(); // why is this needed?
 160   auto tags_iter = o->find("Tag");
 161   while (auto tag_xml = tags_iter.get_next()){
 162     std::string _key,_val;
 163     RGWXMLDecoder::decode_xml("Key", _key, tag_xml);
 164     RGWXMLDecoder::decode_xml("Value", _val, tag_xml);
 165     obj_tags.emplace_tag(std::move(_key), std::move(_val));
 166   }
 167 }
 168
 169 void LCTransition_S3::decode_xml(XMLObj *obj)
 170 {
 171   bool has_days = RGWXMLDecoder::decode_xml("Days", days, obj);
 172   bool has_date = RGWXMLDecoder::decode_xml("Date", date, obj);
 173   if ((has_days && has_date) || (!has_days && !has_date)) {
 174     throw RGWXMLDecoder::err("bad Transition section");
 175   }
 176
 177   if (has_date && !check_date(date)) {
 178     //We need return xml error according to S3
 179     throw RGWXMLDecoder::err("bad Date in Transition section");
 180   }
 181
 182   if (!RGWXMLDecoder::decode_xml("StorageClass", storage_class, obj)) {
 183     throw RGWXMLDecoder::err("missing StorageClass in Transition section");
 184   }
 185 }
 186
 187 void LCTransition_S3::dump_xml(Formatter *f) const {
 188   if (!days.empty()) {
 189     encode_xml("Days", days, f);
 190   } else {
 191     encode_xml("Date", date, f);
 192   }
 193   encode_xml("StorageClass", storage_class, f);
 194 }
 195
 196 void LCNoncurTransition_S3::decode_xml(XMLObj *obj)
 197 {
 198   if (!RGWXMLDecoder::decode_xml("NoncurrentDays", days, obj)) {
 199     throw RGWXMLDecoder::err("missing NoncurrentDays in NoncurrentVersionTransition section");
 200   }
 201   if (!RGWXMLDecoder::decode_xml("StorageClass", storage_class, obj)) {
 202     throw RGWXMLDecoder::err("missing StorageClass in NoncurrentVersionTransition section");
 203   }
 204 }
 205
 206 void LCNoncurTransition_S3::dump_xml(Formatter *f) const
 207 {
 208   encode_xml("NoncurrentDays", days, f);
 209   encode_xml("StorageClass", storage_class, f);
 210 }
 211
 212 void LCRule_S3::decode_xml(XMLObj *obj)
 213 {
 214   id.clear();
 215   prefix.clear();
 216   status.clear();
 217   dm_expiration = false;
 218
 219   RGWXMLDecoder::decode_xml("ID", id, obj);
 220
 221   LCFilter_S3 filter_s3;
 222   if (!RGWXMLDecoder::decode_xml("Filter", filter_s3, obj)) {
 223     // Ideally the following code should be deprecated and we should return
 224     // False here, The new S3 LC configuration xml spec. makes Filter mandatory
 225     // and Prefix optional. However older clients including boto2 still generate
 226     // xml according to the older spec, where Prefix existed outside of Filter
 227     // and S3 itself seems to be sloppy on enforcing the mandatory Filter
 228     // argument. A day will come when S3 enforces their own xml-spec, but it is
 229     // not this day
 230
 231     if (!RGWXMLDecoder::decode_xml("Prefix", prefix, obj)) {
 232       throw RGWXMLDecoder::err("missing Prefix in Filter");
 233     }
 234   }
 235   filter = (LCFilter)filter_s3;
 236
 237   if (!RGWXMLDecoder::decode_xml("Status", status, obj)) {
 238     throw RGWXMLDecoder::err("missing Status in Filter");
 239   }
 240   if (status.compare("Enabled") != 0 && status.compare("Disabled") != 0) {
 241     throw RGWXMLDecoder::err("bad Status in Filter");
 242   }
 243
 244   LCExpiration_S3 s3_expiration;
 245   LCNoncurExpiration_S3 s3_noncur_expiration;
 246   LCMPExpiration_S3 s3_mp_expiration;
 247   LCFilter_S3 s3_filter;
 248
 249   bool has_expiration = RGWXMLDecoder::decode_xml("Expiration", s3_expiration, obj);
 250   bool has_noncur_expiration = RGWXMLDecoder::decode_xml("NoncurrentVersionExpiration", s3_noncur_expiration, obj);
 251   bool has_mp_expiration = RGWXMLDecoder::decode_xml("AbortIncompleteMultipartUpload", s3_mp_expiration, obj);
 252
 253   vector<LCTransition_S3> transitions;
 254   vector<LCNoncurTransition_S3> noncur_transitions;
 255
 256   bool has_transition = RGWXMLDecoder::decode_xml("Transition", transitions, obj);
 257   bool has_noncur_transition = RGWXMLDecoder::decode_xml("NoncurrentVersionTransition", noncur_transitions, obj);
 258
 259   if (!has_expiration &&
 260       !has_noncur_expiration &&
 261       !has_mp_expiration &&
 262       !has_transition &&
 263       !has_noncur_transition) {
 264     throw RGWXMLDecoder::err("bad Rule");
 265   }
 266
 267   if (has_expiration) {
 268     if (s3_expiration.has_days() ||
 269         s3_expiration.has_date()) {
 270       expiration = s3_expiration;
 271     } else {
 272       dm_expiration = s3_expiration.get_dm_expiration();
 273     }
 274   }
 275   if (has_noncur_expiration) {
 276     noncur_expiration = s3_noncur_expiration;
 277   }
 278   if (has_mp_expiration) {
 279     mp_expiration = s3_mp_expiration;
 280   }
 281   for (auto& t : transitions) {
 282     if (!add_transition(t)) {
 283       throw RGWXMLDecoder::err("Failed to add transition");
 284     }
 285   }
 286   for (auto& t : noncur_transitions) {
 287     if (!add_noncur_transition(t)) {
 288       throw RGWXMLDecoder::err("Failed to add non-current version transition");
 289     }
 290   }
 291 }
 292
 293 void LCRule_S3::dump_xml(Formatter *f) const {
 294   encode_xml("ID", id, f);
 295   // In case of an empty filter and an empty Prefix, we defer to Prefix.
 296   if (!filter.empty()) {
 297     const LCFilter_S3& lc_filter = static_cast<const LCFilter_S3&>(filter);
 298     encode_xml("Filter", lc_filter, f);
 299   } else {
 300     encode_xml("Prefix", prefix, f);
 301   }
 302   encode_xml("Status", status, f);
 303   if (!expiration.empty() || dm_expiration) {
 304     LCExpiration_S3 expir(expiration.get_days_str(), expiration.get_date(), dm_expiration);
 305     encode_xml("Expiration", expir, f);
 306   }
 307   if (!noncur_expiration.empty()) {
 308     const LCNoncurExpiration_S3& noncur_expir = static_cast<const LCNoncurExpiration_S3&>(noncur_expiration);
 309     encode_xml("NoncurrentVersionExpiration", noncur_expir, f);
 310   }
 311   if (!mp_expiration.empty()) {
 312     const LCMPExpiration_S3& mp_expir = static_cast<const LCMPExpiration_S3&>(mp_expiration);
 313     encode_xml("AbortIncompleteMultipartUpload", mp_expir, f);
 314   }
 315   if (!transitions.empty()) {
 316     for (auto &elem : transitions) {
 317       const LCTransition_S3& tran = static_cast<const LCTransition_S3&>(elem.second);
 318       encode_xml("Transition", tran, f);
 319     }
 320   }
 321   if (!noncur_transitions.empty()) {
 322     for (auto &elem : noncur_transitions) {
 323       const LCNoncurTransition_S3& noncur_tran = static_cast<const LCNoncurTransition_S3&>(elem.second);
 324       encode_xml("NoncurrentVersionTransition", noncur_tran, f);
 325     }
 326   }
 327 }
 328
 329 int RGWLifecycleConfiguration_S3::rebuild(RGWLifecycleConfiguration& dest)
 330 {
 331   int ret = 0;
 332   multimap<string, LCRule>::iterator iter;
 333   for (iter = rule_map.begin(); iter != rule_map.end(); ++iter) {
 334     LCRule& src_rule = iter->second;
 335     ret = dest.check_and_add_rule(src_rule);
 336     if (ret < 0)
 337       return ret;
 338   }
 339   if (!dest.valid()) {
 340     ret = -ERR_INVALID_REQUEST;
 341   }
 342   return ret;
 343 }
 344
 345
 346 void RGWLifecycleConfiguration_S3::dump_xml(Formatter *f) const
 347 {
 348   for (auto iter = rule_map.begin(); iter != rule_map.end(); ++iter) {
 349     const LCRule_S3& rule = static_cast<const LCRule_S3&>(iter->second);
 350     encode_xml("Rule", rule, f);
 351   }
 352 }
 353