ceph/src/rgw/rgw_rest_s3.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab ft=cpp
   3
   4 #include <errno.h>
   5 #include <array>
   6 #include <string.h>
   7
   8 #include "common/ceph_crypto.h"
   9 #include "common/split.h"
  10 #include "common/Formatter.h"
  11 #include "common/utf8.h"
  12 #include "common/ceph_json.h"
  13 #include "common/safe_io.h"
  14 #include "common/errno.h"
  15 #include "auth/Crypto.h"
  16 #include <boost/algorithm/string.hpp>
  17 #include <boost/algorithm/string/replace.hpp>
  18 #include <boost/utility/string_view.hpp>
  19 #include <boost/tokenizer.hpp>
  20
  21 #include <liboath/oath.h>
  22
  23 #include "rgw_rest.h"
  24 #include "rgw_rest_s3.h"
  25 #include "rgw_rest_s3website.h"
  26 #include "rgw_rest_pubsub.h"
  27 #include "rgw_auth_s3.h"
  28 #include "rgw_acl.h"
  29 #include "rgw_policy_s3.h"
  30 #include "rgw_user.h"
  31 #include "rgw_cors.h"
  32 #include "rgw_cors_s3.h"
  33 #include "rgw_tag_s3.h"
  34
  35 #include "rgw_client_io.h"
  36
  37 #include "rgw_keystone.h"
  38 #include "rgw_auth_keystone.h"
  39 #include "rgw_auth_registry.h"
  40
  41 #include "rgw_es_query.h"
  42
  43 #include <typeinfo> // for 'typeid'
  44
  45 #include "rgw_ldap.h"
  46 #include "rgw_token.h"
  47 #include "rgw_rest_role.h"
  48 #include "rgw_crypt.h"
  49 #include "rgw_crypt_sanitize.h"
  50 #include "rgw_rest_user_policy.h"
  51 #include "rgw_zone.h"
  52 #include "rgw_bucket_sync.h"
  53
  54 #include "services/svc_zone.h"
  55 #include "services/svc_cls.h"
  56
  57 #include "include/ceph_assert.h"
  58 #include "rgw_role.h"
  59 #include "rgw_rest_sts.h"
  60 #include "rgw_rest_iam.h"
  61 #include "rgw_sts.h"
  62
  63 #define dout_context g_ceph_context
  64 #define dout_subsys ceph_subsys_rgw
  65
  66 using namespace rgw;
  67 using namespace ceph::crypto;
  68
  69 using std::get;
  70
  71 void list_all_buckets_start(struct req_state *s)
  72 {
  73   s->formatter->open_array_section_in_ns("ListAllMyBucketsResult", XMLNS_AWS_S3);
  74 }
  75
  76 void list_all_buckets_end(struct req_state *s)
  77 {
  78   s->formatter->close_section();
  79 }
  80
  81 void dump_bucket(struct req_state *s, rgw::sal::RGWBucket& obj)
  82 {
  83   s->formatter->open_object_section("Bucket");
  84   s->formatter->dump_string("Name", obj.get_name());
  85   dump_time(s, "CreationDate", &obj.get_creation_time());
  86   s->formatter->close_section();
  87 }
  88
  89 void rgw_get_errno_s3(rgw_http_error *e , int err_no)
  90 {
  91   rgw_http_errors::const_iterator r = rgw_http_s3_errors.find(err_no);
  92
  93   if (r != rgw_http_s3_errors.end()) {
  94     e->http_ret = r->second.first;
  95     e->s3_code = r->second.second;
  96   } else {
  97     e->http_ret = 500;
  98     e->s3_code = "UnknownError";
  99   }
 100 }
 101
 102 static inline std::string get_s3_expiration_header(
 103   struct req_state* s,
 104   const ceph::real_time& mtime)
 105 {
 106   return rgw::lc::s3_expiration_header(
 107     s, s->object, s->tagset, mtime, s->bucket_attrs);
 108 }
 109
 110 static inline bool get_s3_multipart_abort_header(
 111   struct req_state* s, const ceph::real_time& mtime,
 112   ceph::real_time& date, std::string& rule_id)
 113 {
 114   return rgw::lc::s3_multipart_abort_header(
 115           s, s->object, mtime, s->bucket_attrs, date, rule_id);
 116 }
 117
 118 struct response_attr_param {
 119   const char *param;
 120   const char *http_attr;
 121 };
 122
 123 static struct response_attr_param resp_attr_params[] = {
 124   {"response-content-type", "Content-Type"},
 125   {"response-content-language", "Content-Language"},
 126   {"response-expires", "Expires"},
 127   {"response-cache-control", "Cache-Control"},
 128   {"response-content-disposition", "Content-Disposition"},
 129   {"response-content-encoding", "Content-Encoding"},
 130   {NULL, NULL},
 131 };
 132
 133 int RGWGetObj_ObjStore_S3Website::send_response_data(bufferlist& bl, off_t bl_ofs, off_t bl_len) {
 134   map<string, bufferlist>::iterator iter;
 135   iter = attrs.find(RGW_ATTR_AMZ_WEBSITE_REDIRECT_LOCATION);
 136   if (iter != attrs.end()) {
 137     bufferlist &bl = iter->second;
 138     s->redirect = bl.c_str();
 139     s->err.http_ret = 301;
 140     ldpp_dout(this, 20) << __CEPH_ASSERT_FUNCTION << " redirecting per x-amz-website-redirect-location=" << s->redirect << dendl;
 141     op_ret = -ERR_WEBSITE_REDIRECT;
 142     set_req_state_err(s, op_ret);
 143     dump_errno(s);
 144     dump_content_length(s, 0);
 145     dump_redirect(s, s->redirect);
 146     end_header(s, this);
 147     return op_ret;
 148   } else {
 149     return RGWGetObj_ObjStore_S3::send_response_data(bl, bl_ofs, bl_len);
 150   }
 151 }
 152
 153 int RGWGetObj_ObjStore_S3Website::send_response_data_error()
 154 {
 155   return RGWGetObj_ObjStore_S3::send_response_data_error();
 156 }
 157
 158 int RGWGetObj_ObjStore_S3::get_params()
 159 {
 160   // for multisite sync requests, only read the slo manifest itself, rather than
 161   // all of the data from its parts. the parts will sync as separate objects
 162   skip_manifest = s->info.args.exists(RGW_SYS_PARAM_PREFIX "sync-manifest");
 163
 164   // multisite sync requests should fetch encrypted data, along with the
 165   // attributes needed to support decryption on the other zone
 166   if (s->system_request) {
 167     skip_decrypt = s->info.args.exists(RGW_SYS_PARAM_PREFIX "skip-decrypt");
 168   }
 169
 170   return RGWGetObj_ObjStore::get_params();
 171 }
 172
 173 int RGWGetObj_ObjStore_S3::send_response_data_error()
 174 {
 175   bufferlist bl;
 176   return send_response_data(bl, 0 , 0);
 177 }
 178
 179 template <class T>
 180 int decode_attr_bl_single_value(map<string, bufferlist>& attrs, const char *attr_name, T *result, T def_val)
 181 {
 182   map<string, bufferlist>::iterator iter = attrs.find(attr_name);
 183   if (iter == attrs.end()) {
 184     *result = def_val;
 185     return 0;
 186   }
 187   bufferlist& bl = iter->second;
 188   if (bl.length() == 0) {
 189     *result = def_val;
 190     return 0;
 191   }
 192   auto bliter = bl.cbegin();
 193   try {
 194     decode(*result, bliter);
 195   } catch (buffer::error& err) {
 196     return -EIO;
 197   }
 198   return 0;
 199 }
 200
 201 inline bool str_has_cntrl(const std::string s) {
 202   return std::any_of(s.begin(), s.end(), ::iscntrl);
 203 }
 204
 205 inline bool str_has_cntrl(const char* s) {
 206   std::string _s(s);
 207   return str_has_cntrl(_s);
 208 }
 209
 210 int RGWGetObj_ObjStore_S3::send_response_data(bufferlist& bl, off_t bl_ofs,
 211                                               off_t bl_len)
 212 {
 213   const char *content_type = NULL;
 214   string content_type_str;
 215   map<string, string> response_attrs;
 216   map<string, string>::iterator riter;
 217   bufferlist metadata_bl;
 218
 219   string expires = get_s3_expiration_header(s, lastmod);
 220
 221   if (sent_header)
 222     goto send_data;
 223
 224   if (custom_http_ret) {
 225     set_req_state_err(s, 0);
 226     dump_errno(s, custom_http_ret);
 227   } else {
 228     set_req_state_err(s, (partial_content && !op_ret) ? STATUS_PARTIAL_CONTENT
 229                   : op_ret);
 230     dump_errno(s);
 231   }
 232
 233   if (op_ret)
 234     goto done;
 235
 236   if (range_str)
 237     dump_range(s, start, end, s->obj_size);
 238
 239   if (s->system_request &&
 240       s->info.args.exists(RGW_SYS_PARAM_PREFIX "prepend-metadata")) {
 241
 242     dump_header(s, "Rgwx-Object-Size", (long long)total_len);
 243
 244     if (rgwx_stat) {
 245       /*
 246        * in this case, we're not returning the object's content, only the prepended
 247        * extra metadata
 248        */
 249       total_len = 0;
 250     }
 251
 252     /* JSON encode object metadata */
 253     JSONFormatter jf;
 254     jf.open_object_section("obj_metadata");
 255     encode_json("attrs", attrs, &jf);
 256     utime_t ut(lastmod);
 257     encode_json("mtime", ut, &jf);
 258     jf.close_section();
 259     stringstream ss;
 260     jf.flush(ss);
 261     metadata_bl.append(ss.str());
 262     dump_header(s, "Rgwx-Embedded-Metadata-Len", metadata_bl.length());
 263     total_len += metadata_bl.length();
 264   }
 265
 266   if (s->system_request && !real_clock::is_zero(lastmod)) {
 267     /* we end up dumping mtime in two different methods, a bit redundant */
 268     dump_epoch_header(s, "Rgwx-Mtime", lastmod);
 269     uint64_t pg_ver = 0;
 270     int r = decode_attr_bl_single_value(attrs, RGW_ATTR_PG_VER, &pg_ver, (uint64_t)0);
 271     if (r < 0) {
 272       ldpp_dout(this, 0) << "ERROR: failed to decode pg ver attr, ignoring" << dendl;
 273     }
 274     dump_header(s, "Rgwx-Obj-PG-Ver", pg_ver);
 275
 276     uint32_t source_zone_short_id = 0;
 277     r = decode_attr_bl_single_value(attrs, RGW_ATTR_SOURCE_ZONE, &source_zone_short_id, (uint32_t)0);
 278     if (r < 0) {
 279       ldpp_dout(this, 0) << "ERROR: failed to decode pg ver attr, ignoring" << dendl;
 280     }
 281     if (source_zone_short_id != 0) {
 282       dump_header(s, "Rgwx-Source-Zone-Short-Id", source_zone_short_id);
 283     }
 284   }
 285
 286   for (auto &it : crypt_http_responses)
 287     dump_header(s, it.first, it.second);
 288
 289   dump_content_length(s, total_len);
 290   dump_last_modified(s, lastmod);
 291   dump_header_if_nonempty(s, "x-amz-version-id", version_id);
 292   dump_header_if_nonempty(s, "x-amz-expiration", expires);
 293
 294   if (attrs.find(RGW_ATTR_APPEND_PART_NUM) != attrs.end()) {
 295     dump_header(s, "x-rgw-object-type", "Appendable");
 296     dump_header(s, "x-rgw-next-append-position", s->obj_size);
 297   } else {
 298     dump_header(s, "x-rgw-object-type", "Normal");
 299   }
 300
 301   if (! op_ret) {
 302     if (! lo_etag.empty()) {
 303       /* Handle etag of Swift API's large objects (DLO/SLO). It's entirerly
 304        * legit to perform GET on them through S3 API. In such situation,
 305        * a client should receive the composited content with corresponding
 306        * etag value. */
 307       dump_etag(s, lo_etag);
 308     } else {
 309       auto iter = attrs.find(RGW_ATTR_ETAG);
 310       if (iter != attrs.end()) {
 311         dump_etag(s, iter->second.to_str());
 312       }
 313     }
 314
 315     for (struct response_attr_param *p = resp_attr_params; p->param; p++) {
 316       bool exists;
 317       string val = s->info.args.get(p->param, &exists);
 318       if (exists) {
 319         /* reject unauthenticated response header manipulation, see
 320          * https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html */
 321         if (s->auth.identity->is_anonymous()) {
 322           return -ERR_INVALID_REQUEST;
 323         }
 324         /* HTTP specification says no control characters should be present in
 325          * header values: https://tools.ietf.org/html/rfc7230#section-3.2
 326          *      field-vchar    = VCHAR / obs-text
 327          *
 328          * Failure to validate this permits a CRLF injection in HTTP headers,
 329          * whereas S3 GetObject only permits specific headers.
 330          */
 331         if(str_has_cntrl(val)) {
 332           /* TODO: return a more distinct error in future;
 333            * stating what the problem is */
 334           return -ERR_INVALID_REQUEST;
 335         }
 336
 337         if (strcmp(p->param, "response-content-type") != 0) {
 338           response_attrs[p->http_attr] = val;
 339         } else {
 340           content_type_str = val;
 341           content_type = content_type_str.c_str();
 342         }
 343       }
 344     }
 345
 346     for (auto iter = attrs.begin(); iter != attrs.end(); ++iter) {
 347       const char *name = iter->first.c_str();
 348       map<string, string>::iterator aiter = rgw_to_http_attrs.find(name);
 349       if (aiter != rgw_to_http_attrs.end()) {
 350         if (response_attrs.count(aiter->second) == 0) {
 351           /* Was not already overridden by a response param. */
 352
 353           size_t len = iter->second.length();
 354           string s(iter->second.c_str(), len);
 355           while (len && !s[len - 1]) {
 356             --len;
 357             s.resize(len);
 358           }
 359           response_attrs[aiter->second] = s;
 360         }
 361       } else if (iter->first.compare(RGW_ATTR_CONTENT_TYPE) == 0) {
 362         /* Special handling for content_type. */
 363         if (!content_type) {
 364           content_type_str = rgw_bl_str(iter->second);
 365           content_type = content_type_str.c_str();
 366         }
 367       } else if (strcmp(name, RGW_ATTR_SLO_UINDICATOR) == 0) {
 368         // this attr has an extra length prefix from encode() in prior versions
 369         dump_header(s, "X-Object-Meta-Static-Large-Object", "True");
 370       } else if (strncmp(name, RGW_ATTR_META_PREFIX,
 371                          sizeof(RGW_ATTR_META_PREFIX)-1) == 0) {
 372         /* User custom metadata. */
 373         name += sizeof(RGW_ATTR_PREFIX) - 1;
 374         dump_header(s, name, iter->second);
 375       } else if (iter->first.compare(RGW_ATTR_TAGS) == 0) {
 376         RGWObjTags obj_tags;
 377         try{
 378           auto it = iter->second.cbegin();
 379           obj_tags.decode(it);
 380         } catch (buffer::error &err) {
 381           ldpp_dout(this,0) << "Error caught buffer::error couldn't decode TagSet " << dendl;
 382         }
 383         dump_header(s, RGW_AMZ_TAG_COUNT, obj_tags.count());
 384       } else if (iter->first.compare(RGW_ATTR_OBJECT_RETENTION) == 0 && get_retention){
 385         RGWObjectRetention retention;
 386         try {
 387           decode(retention, iter->second);
 388           dump_header(s, "x-amz-object-lock-mode", retention.get_mode());
 389           dump_time_header(s, "x-amz-object-lock-retain-until-date", retention.get_retain_until_date());
 390         } catch (buffer::error& err) {
 391           ldpp_dout(this, 0) << "ERROR: failed to decode RGWObjectRetention" << dendl;
 392         }
 393       } else if (iter->first.compare(RGW_ATTR_OBJECT_LEGAL_HOLD) == 0 && get_legal_hold) {
 394         RGWObjectLegalHold legal_hold;
 395         try {
 396           decode(legal_hold, iter->second);
 397           dump_header(s, "x-amz-object-lock-legal-hold",legal_hold.get_status());
 398         } catch (buffer::error& err) {
 399           ldpp_dout(this, 0) << "ERROR: failed to decode RGWObjectLegalHold" << dendl;
 400         }
 401       }
 402     }
 403   }
 404
 405 done:
 406   for (riter = response_attrs.begin(); riter != response_attrs.end();
 407        ++riter) {
 408     dump_header(s, riter->first, riter->second);
 409   }
 410
 411   if (op_ret == -ERR_NOT_MODIFIED) {
 412       end_header(s, this);
 413   } else {
 414       if (!content_type)
 415           content_type = "binary/octet-stream";
 416
 417       end_header(s, this, content_type);
 418   }
 419
 420   if (metadata_bl.length()) {
 421     dump_body(s, metadata_bl);
 422   }
 423   sent_header = true;
 424
 425 send_data:
 426   if (get_data && !op_ret) {
 427     int r = dump_body(s, bl.c_str() + bl_ofs, bl_len);
 428     if (r < 0)
 429       return r;
 430   }
 431
 432   return 0;
 433 }
 434
 435 int RGWGetObj_ObjStore_S3::get_decrypt_filter(std::unique_ptr<RGWGetObj_Filter> *filter, RGWGetObj_Filter* cb, bufferlist* manifest_bl)
 436 {
 437   if (skip_decrypt) { // bypass decryption for multisite sync requests
 438     return 0;
 439   }
 440
 441   int res = 0;
 442   std::unique_ptr<BlockCrypt> block_crypt;
 443   res = rgw_s3_prepare_decrypt(s, attrs, &block_crypt, crypt_http_responses);
 444   if (res == 0) {
 445     if (block_crypt != nullptr) {
 446       auto f = std::make_unique<RGWGetObj_BlockDecrypt>(s->cct, cb, std::move(block_crypt));
 447       if (manifest_bl != nullptr) {
 448         res = f->read_manifest(*manifest_bl);
 449         if (res == 0) {
 450           *filter = std::move(f);
 451         }
 452       }
 453     }
 454   }
 455   return res;
 456 }
 457 int RGWGetObj_ObjStore_S3::verify_requester(const rgw::auth::StrategyRegistry& auth_registry)
 458 {
 459   int ret = -EINVAL;
 460   ret = RGWOp::verify_requester(auth_registry);
 461   if(!s->user->get_caps().check_cap("amz-cache", RGW_CAP_READ) && !ret && s->info.env->exists("HTTP_X_AMZ_CACHE"))
 462     ret = override_range_hdr(auth_registry);
 463   return ret;
 464 }
 465 int RGWGetObj_ObjStore_S3::override_range_hdr(const rgw::auth::StrategyRegistry& auth_registry)
 466 {
 467   int ret = -EINVAL;
 468   ldpp_dout(this, 10) << "cache override headers" << dendl;
 469   RGWEnv* rgw_env = const_cast<RGWEnv *>(s->info.env);
 470   const char* backup_range = rgw_env->get("HTTP_RANGE");
 471   const char hdrs_split[2] = {(char)178,'\0'};
 472   const char kv_split[2] = {(char)177,'\0'};
 473   const char* cache_hdr = rgw_env->get("HTTP_X_AMZ_CACHE");
 474   for (std::string_view hdr : ceph::split(cache_hdr, hdrs_split)) {
 475     auto kv = ceph::split(hdr, kv_split);
 476     auto k = kv.begin();
 477     if (std::distance(k, kv.end()) != 2) {
 478       return -EINVAL;
 479     }
 480     auto v = std::next(k);
 481     std::string key = "HTTP_";
 482     key.append(*k);
 483     boost::replace_all(key, "-", "_");
 484     rgw_env->set(std::move(key), std::string(*v));
 485     ldpp_dout(this, 10) << "after splitting cache kv key: " << key  << " " << rgw_env->get(key.c_str())  << dendl;
 486   }
 487   ret = RGWOp::verify_requester(auth_registry);
 488   if(!ret && backup_range) {
 489     rgw_env->set("HTTP_RANGE",backup_range);
 490   } else {
 491     rgw_env->remove("HTTP_RANGE");
 492   }
 493   return ret;
 494 }
 495
 496
 497 void RGWGetObjTags_ObjStore_S3::send_response_data(bufferlist& bl)
 498 {
 499   dump_errno(s);
 500   end_header(s, this, "application/xml");
 501   dump_start(s);
 502
 503   s->formatter->open_object_section_in_ns("Tagging", XMLNS_AWS_S3);
 504   s->formatter->open_object_section("TagSet");
 505   if (has_tags){
 506     RGWObjTagSet_S3 tagset;
 507     auto iter = bl.cbegin();
 508     try {
 509       tagset.decode(iter);
 510     } catch (buffer::error& err) {
 511       ldpp_dout(this,0) << "ERROR: caught buffer::error, couldn't decode TagSet" << dendl;
 512       op_ret= -EIO;
 513       return;
 514     }
 515     tagset.dump_xml(s->formatter);
 516   }
 517   s->formatter->close_section();
 518   s->formatter->close_section();
 519   rgw_flush_formatter_and_reset(s, s->formatter);
 520 }
 521
 522
 523 int RGWPutObjTags_ObjStore_S3::get_params()
 524 {
 525   RGWXMLParser parser;
 526
 527   if (!parser.init()){
 528     return -EINVAL;
 529   }
 530
 531   const auto max_size = s->cct->_conf->rgw_max_put_param_size;
 532
 533   int r = 0;
 534   bufferlist data;
 535   std::tie(r, data) = rgw_rest_read_all_input(s, max_size, false);
 536
 537   if (r < 0)
 538     return r;
 539
 540   if (!parser.parse(data.c_str(), data.length(), 1)) {
 541     return -ERR_MALFORMED_XML;
 542   }
 543
 544   RGWObjTagging_S3 tagging;
 545
 546   try {
 547     RGWXMLDecoder::decode_xml("Tagging", tagging, &parser);
 548   } catch (RGWXMLDecoder::err& err) {
 549     ldpp_dout(this, 5) << "Malformed tagging request: " << err << dendl;
 550     return -ERR_MALFORMED_XML;
 551   }
 552
 553   RGWObjTags obj_tags;
 554   r = tagging.rebuild(obj_tags);
 555   if (r < 0)
 556     return r;
 557
 558   obj_tags.encode(tags_bl);
 559   ldpp_dout(this, 20) << "Read " << obj_tags.count() << "tags" << dendl;
 560
 561   return 0;
 562 }
 563
 564 void RGWPutObjTags_ObjStore_S3::send_response()
 565 {
 566   if (op_ret)
 567     set_req_state_err(s, op_ret);
 568   dump_errno(s);
 569   end_header(s, this, "application/xml");
 570   dump_start(s);
 571
 572 }
 573
 574 void RGWDeleteObjTags_ObjStore_S3::send_response()
 575 {
 576   int r = op_ret;
 577   if (r == -ENOENT)
 578     r = 0;
 579   if (!r)
 580     r = STATUS_NO_CONTENT;
 581
 582   set_req_state_err(s, r);
 583   dump_errno(s);
 584   end_header(s, this);
 585 }
 586
 587 void RGWGetBucketTags_ObjStore_S3::send_response_data(bufferlist& bl)
 588 {
 589   if (op_ret)
 590     set_req_state_err(s, op_ret);
 591   dump_errno(s);
 592   end_header(s, this, "application/xml");
 593   dump_start(s);
 594
 595   if (!op_ret) {
 596   s->formatter->open_object_section_in_ns("Tagging", XMLNS_AWS_S3);
 597   s->formatter->open_object_section("TagSet");
 598   if (has_tags){
 599     RGWObjTagSet_S3 tagset;
 600     auto iter = bl.cbegin();
 601     try {
 602       tagset.decode(iter);
 603     } catch (buffer::error& err) {
 604       ldout(s->cct,0) << "ERROR: caught buffer::error, couldn't decode TagSet" << dendl;
 605       op_ret= -EIO;
 606       return;
 607     }
 608     tagset.dump_xml(s->formatter);
 609   }
 610   s->formatter->close_section();
 611   s->formatter->close_section();
 612   rgw_flush_formatter_and_reset(s, s->formatter);
 613   }
 614 }
 615
 616 int RGWPutBucketTags_ObjStore_S3::get_params()
 617 {
 618   RGWXMLParser parser;
 619
 620   if (!parser.init()){
 621     return -EINVAL;
 622   }
 623
 624   const auto max_size = s->cct->_conf->rgw_max_put_param_size;
 625   int r = 0;
 626   bufferlist data;
 627
 628   std::tie(r, data) = rgw_rest_read_all_input(s, max_size, false);
 629
 630   if (r < 0)
 631     return r;
 632
 633   if (!parser.parse(data.c_str(), data.length(), 1)) {
 634     return -ERR_MALFORMED_XML;
 635   }
 636
 637   RGWObjTagging_S3 tagging;
 638   try {
 639     RGWXMLDecoder::decode_xml("Tagging", tagging, &parser);
 640   } catch (RGWXMLDecoder::err& err) {
 641
 642     ldout(s->cct, 5) << "Malformed tagging request: " << err << dendl;
 643     return -ERR_MALFORMED_XML;
 644   }
 645
 646   RGWObjTags obj_tags(50); // A tag set can contain as many as 50 tags, or it can be empty.
 647   r = tagging.rebuild(obj_tags);
 648   if (r < 0)
 649     return r;
 650
 651   obj_tags.encode(tags_bl);
 652   ldout(s->cct, 20) << "Read " << obj_tags.count() << "tags" << dendl;
 653
 654   // forward bucket tags requests to meta master zone
 655   if (!store->svc()->zone->is_meta_master()) {
 656     /* only need to keep this data around if we're not meta master */
 657     in_data = std::move(data);
 658   }
 659
 660   return 0;
 661 }
 662
 663 void RGWPutBucketTags_ObjStore_S3::send_response()
 664 {
 665   if (op_ret)
 666     set_req_state_err(s, op_ret);
 667   dump_errno(s);
 668   end_header(s, this, "application/xml");
 669   dump_start(s);
 670 }
 671
 672 void RGWDeleteBucketTags_ObjStore_S3::send_response()
 673 {
 674   if (op_ret)
 675     set_req_state_err(s, op_ret);
 676   dump_errno(s);
 677   end_header(s, this, "application/xml");
 678   dump_start(s);
 679 }
 680
 681 namespace {
 682
 683 bool is_valid_status(const string& s) {
 684   return (s == "Enabled" ||
 685           s == "Disabled");
 686 }
 687
 688 static string enabled_group_id = "s3-bucket-replication:enabled";
 689 static string disabled_group_id = "s3-bucket-replication:disabled";
 690
 691 struct ReplicationConfiguration {
 692   string role;
 693
 694   struct Rule {
 695     struct DeleteMarkerReplication {
 696       string status;
 697
 698       void decode_xml(XMLObj *obj) {
 699         RGWXMLDecoder::decode_xml("Status", status, obj);
 700       }
 701
 702       void dump_xml(Formatter *f) const {
 703         encode_xml("Status", status, f);
 704       }
 705
 706       bool is_valid(CephContext *cct) const {
 707         bool result = is_valid_status(status);
 708         if (!result) {
 709           ldout(cct, 5) << "NOTICE: bad status provided in DeleteMarkerReplication element (status=" << status << ")" << dendl;
 710         }
 711         return result;
 712       }
 713     };
 714
 715     struct Source { /* rgw extension */
 716       std::vector<string> zone_names;
 717
 718       void decode_xml(XMLObj *obj) {
 719         RGWXMLDecoder::decode_xml("Zone", zone_names, obj);
 720       }
 721
 722       void dump_xml(Formatter *f) const {
 723         encode_xml("Zone", zone_names, f);
 724       }
 725     };
 726
 727     struct Destination {
 728       struct AccessControlTranslation {
 729         string owner;
 730
 731         void decode_xml(XMLObj *obj) {
 732           RGWXMLDecoder::decode_xml("Owner", owner, obj);
 733         }
 734         void dump_xml(Formatter *f) const {
 735           encode_xml("Owner", owner, f);
 736         }
 737       };
 738
 739       std::optional<AccessControlTranslation> acl_translation;
 740       std::optional<string> account;
 741       string bucket;
 742       std::optional<string> storage_class;
 743       std::vector<string> zone_names;
 744
 745       void decode_xml(XMLObj *obj) {
 746         RGWXMLDecoder::decode_xml("AccessControlTranslation", acl_translation, obj);
 747         RGWXMLDecoder::decode_xml("Account", account, obj);
 748         if (account && account->empty()) {
 749           account.reset();
 750         }
 751         RGWXMLDecoder::decode_xml("Bucket", bucket, obj);
 752         RGWXMLDecoder::decode_xml("StorageClass", storage_class, obj);
 753         if (storage_class && storage_class->empty()) {
 754           storage_class.reset();
 755         }
 756         RGWXMLDecoder::decode_xml("Zone", zone_names, obj); /* rgw extension */
 757       }
 758
 759       void dump_xml(Formatter *f) const {
 760         encode_xml("AccessControlTranslation", acl_translation, f);
 761         encode_xml("Account", account, f);
 762         encode_xml("Bucket", bucket, f);
 763         encode_xml("StorageClass", storage_class, f);
 764         encode_xml("Zone", zone_names, f);
 765       }
 766     };
 767
 768     struct Filter {
 769       struct Tag {
 770         string key;
 771         string value;
 772
 773         bool empty() const {
 774           return key.empty() && value.empty();
 775         }
 776
 777         void decode_xml(XMLObj *obj) {
 778           RGWXMLDecoder::decode_xml("Key", key, obj);
 779           RGWXMLDecoder::decode_xml("Value", value, obj);
 780         };
 781
 782         void dump_xml(Formatter *f) const {
 783           encode_xml("Key", key, f);
 784           encode_xml("Value", value, f);
 785         }
 786       };
 787
 788       struct AndElements {
 789         std::optional<string> prefix;
 790         std::vector<Tag> tags;
 791
 792         bool empty() const {
 793           return !prefix &&
 794             (tags.size() == 0);
 795         }
 796
 797         void decode_xml(XMLObj *obj) {
 798           std::vector<Tag> _tags;
 799           RGWXMLDecoder::decode_xml("Prefix", prefix, obj);
 800           if (prefix && prefix->empty()) {
 801             prefix.reset();
 802           }
 803           RGWXMLDecoder::decode_xml("Tag", _tags, obj);
 804           for (auto& t : _tags) {
 805             if (!t.empty()) {
 806               tags.push_back(std::move(t));
 807             }
 808           }
 809         };
 810
 811         void dump_xml(Formatter *f) const {
 812           encode_xml("Prefix", prefix, f);
 813           encode_xml("Tag", tags, f);
 814         }
 815       };
 816
 817       std::optional<string> prefix;
 818       std::optional<Tag> tag;
 819       std::optional<AndElements> and_elements;
 820
 821       bool empty() const {
 822         return (!prefix && !tag && !and_elements);
 823       }
 824
 825       void decode_xml(XMLObj *obj) {
 826         RGWXMLDecoder::decode_xml("Prefix", prefix, obj);
 827         if (prefix && prefix->empty()) {
 828           prefix.reset();
 829         }
 830         RGWXMLDecoder::decode_xml("Tag", tag, obj);
 831         if (tag && tag->empty()) {
 832           tag.reset();
 833         }
 834         RGWXMLDecoder::decode_xml("And", and_elements, obj);
 835         if (and_elements && and_elements->empty()) {
 836           and_elements.reset();
 837         }
 838       };
 839
 840       void dump_xml(Formatter *f) const {
 841         encode_xml("Prefix", prefix, f);
 842         encode_xml("Tag", tag, f);
 843         encode_xml("And", and_elements, f);
 844       }
 845
 846       bool is_valid(CephContext *cct) const {
 847         if (tag && prefix) {
 848           ldout(cct, 5) << "NOTICE: both tag and prefix were provided in replication filter rule" << dendl;
 849           return false;
 850         }
 851
 852         if (and_elements) {
 853           if (prefix && and_elements->prefix) {
 854             ldout(cct, 5) << "NOTICE: too many prefixes were provided in re" << dendl;
 855             return false;
 856           }
 857         }
 858         return true;
 859       };
 860
 861       int to_sync_pipe_filter(CephContext *cct,
 862                               rgw_sync_pipe_filter *f) const {
 863         if (!is_valid(cct)) {
 864           return -EINVAL;
 865         }
 866         if (prefix) {
 867           f->prefix = *prefix;
 868         }
 869         if (tag) {
 870           f->tags.insert(rgw_sync_pipe_filter_tag(tag->key, tag->value));
 871         }
 872
 873         if (and_elements) {
 874           if (and_elements->prefix) {
 875             f->prefix = *and_elements->prefix;
 876           }
 877           for (auto& t : and_elements->tags) {
 878             f->tags.insert(rgw_sync_pipe_filter_tag(t.key, t.value));
 879           }
 880         }
 881         return 0;
 882       }
 883
 884       void from_sync_pipe_filter(const rgw_sync_pipe_filter& f) {
 885         if (f.prefix && f.tags.empty()) {
 886           prefix = f.prefix;
 887           return;
 888         }
 889         if (f.prefix) {
 890           and_elements.emplace();
 891           and_elements->prefix = f.prefix;
 892         } else if (f.tags.size() == 1) {
 893           auto iter = f.tags.begin();
 894           if (iter == f.tags.end()) {
 895             /* should never happen */
 896             return;
 897           }
 898           auto& t = *iter;
 899           tag.emplace();
 900           tag->key = t.key;
 901           tag->value = t.value;
 902           return;
 903         }
 904
 905         if (f.tags.empty()) {
 906           return;
 907         }
 908
 909         if (!and_elements) {
 910           and_elements.emplace();
 911         }
 912
 913         for (auto& t : f.tags) {
 914           auto& tag = and_elements->tags.emplace_back();
 915           tag.key = t.key;
 916           tag.value = t.value;
 917         }
 918       }
 919     };
 920
 921     set<rgw_zone_id> get_zone_ids_from_names(rgw::sal::RGWRadosStore *store,
 922                                              const vector<string>& zone_names) const {
 923       set<rgw_zone_id> ids;
 924
 925       for (auto& name : zone_names) {
 926         rgw_zone_id id;
 927         if (store->svc()->zone->find_zone_id_by_name(name, &id)) {
 928           ids.insert(std::move(id));
 929         }
 930       }
 931
 932       return ids;
 933     }
 934
 935     vector<string> get_zone_names_from_ids(rgw::sal::RGWRadosStore *store,
 936                                            const set<rgw_zone_id>& zone_ids) const {
 937       vector<string> names;
 938
 939       for (auto& id : zone_ids) {
 940         RGWZone *zone;
 941         if (store->svc()->zone->find_zone(id, &zone)) {
 942           names.emplace_back(zone->name);
 943         }
 944       }
 945
 946       return names;
 947     }
 948
 949     std::optional<DeleteMarkerReplication> delete_marker_replication;
 950     std::optional<Source> source;
 951     Destination destination;
 952     std::optional<Filter> filter;
 953     string id;
 954     int32_t priority;
 955     string status;
 956
 957     void decode_xml(XMLObj *obj) {
 958       RGWXMLDecoder::decode_xml("DeleteMarkerReplication", delete_marker_replication, obj);
 959       RGWXMLDecoder::decode_xml("Source", source, obj);
 960       RGWXMLDecoder::decode_xml("Destination", destination, obj);
 961       RGWXMLDecoder::decode_xml("ID", id, obj);
 962
 963       std::optional<string> prefix;
 964       RGWXMLDecoder::decode_xml("Prefix", prefix, obj);
 965       if (prefix) {
 966         filter.emplace();
 967         filter->prefix = prefix;
 968       }
 969
 970       if (!filter) {
 971         RGWXMLDecoder::decode_xml("Filter", filter, obj);
 972       } else {
 973         /* don't want to have filter reset because it might have been initialized
 974          * when decoding prefix
 975          */
 976         RGWXMLDecoder::decode_xml("Filter", *filter, obj);
 977       }
 978
 979       RGWXMLDecoder::decode_xml("Priority", priority, obj);
 980       RGWXMLDecoder::decode_xml("Status", status, obj);
 981     }
 982
 983     void dump_xml(Formatter *f) const {
 984       encode_xml("DeleteMarkerReplication", delete_marker_replication, f);
 985       encode_xml("Source", source, f);
 986       encode_xml("Destination", destination, f);
 987       encode_xml("Filter", filter, f);
 988       encode_xml("ID", id, f);
 989       encode_xml("Priority", priority, f);
 990       encode_xml("Status", status, f);
 991     }
 992
 993     bool is_valid(CephContext *cct) const {
 994       if (!is_valid_status(status)) {
 995         ldout(cct, 5) << "NOTICE: bad status provided in rule (status=" << status << ")" << dendl;
 996         return false;
 997       }
 998       if ((filter && !filter->is_valid(cct)) ||
 999           (delete_marker_replication && !delete_marker_replication->is_valid(cct))) {
1000         return false;
1001       }
1002       return true;
1003     }
1004
1005     int to_sync_policy_pipe(req_state *s, rgw::sal::RGWRadosStore *store,
1006                             rgw_sync_bucket_pipes *pipe,
1007                             bool *enabled) const {
1008       if (!is_valid(s->cct)) {
1009         return -EINVAL;
1010       }
1011
1012       pipe->id = id;
1013       pipe->params.priority = priority;
1014
1015       const auto& user_id = s->user->get_id();
1016
1017       rgw_bucket_key dest_bk(user_id.tenant,
1018                              destination.bucket);
1019
1020       if (source && !source->zone_names.empty()) {
1021         pipe->source.zones = get_zone_ids_from_names(store, source->zone_names);
1022       } else {
1023         pipe->source.set_all_zones(true);
1024       }
1025       if (!destination.zone_names.empty()) {
1026         pipe->dest.zones = get_zone_ids_from_names(store, destination.zone_names);
1027       } else {
1028         pipe->dest.set_all_zones(true);
1029       }
1030       pipe->dest.bucket.emplace(dest_bk);
1031
1032       if (filter) {
1033         int r = filter->to_sync_pipe_filter(s->cct, &pipe->params.source.filter);
1034         if (r < 0) {
1035           return r;
1036         }
1037       }
1038       if (destination.acl_translation) {
1039         rgw_user u;
1040         u.tenant = user_id.tenant;
1041         u.from_str(destination.acl_translation->owner); /* explicit tenant will override tenant,
1042                                                            otherwise will inherit it from s->user */
1043         pipe->params.dest.acl_translation.emplace();
1044         pipe->params.dest.acl_translation->owner = u;
1045       }
1046       pipe->params.dest.storage_class = destination.storage_class;
1047
1048       *enabled = (status == "Enabled");
1049
1050       pipe->params.mode = rgw_sync_pipe_params::Mode::MODE_USER;
1051       pipe->params.user = user_id.to_str();
1052
1053       return 0;
1054     }
1055
1056     void from_sync_policy_pipe(rgw::sal::RGWRadosStore *store,
1057                               const rgw_sync_bucket_pipes& pipe,
1058                               bool enabled) {
1059       id = pipe.id;
1060       status = (enabled ? "Enabled" : "Disabled");
1061       priority = pipe.params.priority;
1062
1063       if (pipe.source.all_zones) {
1064         source.reset();
1065       } else if (pipe.source.zones) {
1066         source.emplace();
1067         source->zone_names = get_zone_names_from_ids(store, *pipe.source.zones);
1068       }
1069
1070       if (!pipe.dest.all_zones &&
1071           pipe.dest.zones) {
1072         destination.zone_names = get_zone_names_from_ids(store, *pipe.dest.zones);
1073       }
1074
1075       if (pipe.params.dest.acl_translation) {
1076         destination.acl_translation.emplace();
1077         destination.acl_translation->owner = pipe.params.dest.acl_translation->owner.to_str();
1078       }
1079
1080       if (pipe.params.dest.storage_class) {
1081         destination.storage_class = *pipe.params.dest.storage_class;
1082       }
1083
1084       if (pipe.dest.bucket) {
1085         destination.bucket = pipe.dest.bucket->get_key();
1086       }
1087
1088       filter.emplace();
1089       filter->from_sync_pipe_filter(pipe.params.source.filter);
1090
1091       if (filter->empty()) {
1092         filter.reset();
1093       }
1094     }
1095   };
1096
1097   std::vector<Rule> rules;
1098
1099   void decode_xml(XMLObj *obj) {
1100     RGWXMLDecoder::decode_xml("Role", role, obj);
1101     RGWXMLDecoder::decode_xml("Rule", rules, obj);
1102   }
1103
1104   void dump_xml(Formatter *f) const {
1105     encode_xml("Role", role, f);
1106     encode_xml("Rule", rules, f);
1107   }
1108
1109   int to_sync_policy_groups(req_state *s, rgw::sal::RGWRadosStore *store,
1110                             vector<rgw_sync_policy_group> *result) const {
1111     result->resize(2);
1112
1113     rgw_sync_policy_group& enabled_group = (*result)[0];
1114     rgw_sync_policy_group& disabled_group = (*result)[1];
1115
1116     enabled_group.id = enabled_group_id;
1117     enabled_group.status = rgw_sync_policy_group::Status::ENABLED;
1118     disabled_group.id = disabled_group_id;
1119     disabled_group.status = rgw_sync_policy_group::Status::ALLOWED; /* not enabled, not forbidden */
1120
1121     for (auto& rule : rules) {
1122       rgw_sync_bucket_pipes pipe;
1123       bool enabled;
1124       int r = rule.to_sync_policy_pipe(s, store, &pipe, &enabled);
1125       if (r < 0) {
1126         ldout(s->cct, 5) << "NOTICE: failed to convert replication configuration into sync policy pipe (rule.id=" << rule.id << "): " << cpp_strerror(-r) << dendl;
1127         return r;
1128       }
1129
1130       if (enabled) {
1131         enabled_group.pipes.emplace_back(std::move(pipe));
1132       } else {
1133         disabled_group.pipes.emplace_back(std::move(pipe));
1134       }
1135     }
1136     return 0;
1137   }
1138
1139   void from_sync_policy_group(rgw::sal::RGWRadosStore *store,
1140                               const rgw_sync_policy_group& group) {
1141
1142     bool enabled = (group.status == rgw_sync_policy_group::Status::ENABLED);
1143
1144     for (auto& pipe : group.pipes) {
1145       auto& rule = rules.emplace_back();
1146       rule.from_sync_policy_pipe(store, pipe, enabled);
1147     }
1148   }
1149 };
1150
1151 }
1152
1153 void RGWGetBucketReplication_ObjStore_S3::send_response_data()
1154 {
1155   if (op_ret)
1156     set_req_state_err(s, op_ret);
1157   dump_errno(s);
1158   end_header(s, this, "application/xml");
1159   dump_start(s);
1160
1161   ReplicationConfiguration conf;
1162
1163   if (s->bucket_info.sync_policy) {
1164     auto policy = s->bucket_info.sync_policy;
1165
1166     auto iter = policy->groups.find(enabled_group_id);
1167     if (iter != policy->groups.end()) {
1168       conf.from_sync_policy_group(store, iter->second);
1169     }
1170     iter = policy->groups.find(disabled_group_id);
1171     if (iter != policy->groups.end()) {
1172       conf.from_sync_policy_group(store, iter->second);
1173     }
1174   }
1175
1176   if (!op_ret) {
1177   s->formatter->open_object_section_in_ns("ReplicationConfiguration", XMLNS_AWS_S3);
1178   conf.dump_xml(s->formatter);
1179   s->formatter->close_section();
1180   rgw_flush_formatter_and_reset(s, s->formatter);
1181   }
1182 }
1183
1184 int RGWPutBucketReplication_ObjStore_S3::get_params()
1185 {
1186   RGWXMLParser parser;
1187
1188   if (!parser.init()){
1189     return -EINVAL;
1190   }
1191
1192   const auto max_size = s->cct->_conf->rgw_max_put_param_size;
1193   int r = 0;
1194   bufferlist data;
1195
1196   std::tie(r, data) = rgw_rest_read_all_input(s, max_size, false);
1197
1198   if (r < 0)
1199     return r;
1200
1201   if (!parser.parse(data.c_str(), data.length(), 1)) {
1202     return -ERR_MALFORMED_XML;
1203   }
1204
1205   ReplicationConfiguration conf;
1206   try {
1207     RGWXMLDecoder::decode_xml("ReplicationConfiguration", conf, &parser);
1208   } catch (RGWXMLDecoder::err& err) {
1209
1210     ldout(s->cct, 5) << "Malformed tagging request: " << err << dendl;
1211     return -ERR_MALFORMED_XML;
1212   }
1213
1214   r = conf.to_sync_policy_groups(s, store, &sync_policy_groups);
1215   if (r < 0) {
1216     return r;
1217   }
1218
1219   // forward requests to meta master zone
1220   if (!store->svc()->zone->is_meta_master()) {
1221     /* only need to keep this data around if we're not meta master */
1222     in_data = std::move(data);
1223   }
1224
1225   return 0;
1226 }
1227
1228 void RGWPutBucketReplication_ObjStore_S3::send_response()
1229 {
1230   if (op_ret)
1231     set_req_state_err(s, op_ret);
1232   dump_errno(s);
1233   end_header(s, this, "application/xml");
1234   dump_start(s);
1235 }
1236
1237 void RGWDeleteBucketReplication_ObjStore_S3::update_sync_policy(rgw_sync_policy_info *policy)
1238 {
1239   policy->groups.erase(enabled_group_id);
1240   policy->groups.erase(disabled_group_id);
1241 }
1242
1243 void RGWDeleteBucketReplication_ObjStore_S3::send_response()
1244 {
1245   if (op_ret)
1246     set_req_state_err(s, op_ret);
1247   dump_errno(s);
1248   end_header(s, this, "application/xml");
1249   dump_start(s);
1250 }
1251
1252 void RGWListBuckets_ObjStore_S3::send_response_begin(bool has_buckets)
1253 {
1254   if (op_ret)
1255     set_req_state_err(s, op_ret);
1256   dump_errno(s);
1257   dump_start(s);
1258   // Explicitly use chunked transfer encoding so that we can stream the result
1259   // to the user without having to wait for the full length of it.
1260   end_header(s, NULL, "application/xml", CHUNKED_TRANSFER_ENCODING);
1261
1262   if (! op_ret) {
1263     list_all_buckets_start(s);
1264     dump_owner(s, s->user->get_id(), s->user->get_display_name());
1265     s->formatter->open_array_section("Buckets");
1266     sent_data = true;
1267   }
1268 }
1269
1270 void RGWListBuckets_ObjStore_S3::send_response_data(rgw::sal::RGWBucketList& buckets)
1271 {
1272   if (!sent_data)
1273     return;
1274
1275   map<string, rgw::sal::RGWBucket*>& m = buckets.get_buckets();
1276   map<string, rgw::sal::RGWBucket*>::iterator iter;
1277
1278   for (iter = m.begin(); iter != m.end(); ++iter) {
1279     rgw::sal::RGWBucket* obj = iter->second;
1280     dump_bucket(s, *obj);
1281   }
1282   rgw_flush_formatter(s, s->formatter);
1283 }
1284
1285 void RGWListBuckets_ObjStore_S3::send_response_end()
1286 {
1287   if (sent_data) {
1288     s->formatter->close_section();
1289     list_all_buckets_end(s);
1290     rgw_flush_formatter_and_reset(s, s->formatter);
1291   }
1292 }
1293
1294 int RGWGetUsage_ObjStore_S3::get_params()
1295 {
1296   start_date = s->info.args.get("start-date");
1297   end_date = s->info.args.get("end-date");
1298   return 0;
1299 }
1300
1301 static void dump_usage_categories_info(Formatter *formatter, const rgw_usage_log_entry& entry, map<string, bool> *categories)
1302 {
1303   formatter->open_array_section("categories");
1304   map<string, rgw_usage_data>::const_iterator uiter;
1305   for (uiter = entry.usage_map.begin(); uiter != entry.usage_map.end(); ++uiter) {
1306     if (categories && !categories->empty() && !categories->count(uiter->first))
1307       continue;
1308     const rgw_usage_data& usage = uiter->second;
1309     formatter->open_object_section("Entry");
1310     encode_json("Category", uiter->first, formatter);
1311     encode_json("BytesSent", usage.bytes_sent, formatter);
1312     encode_json("BytesReceived", usage.bytes_received, formatter);
1313     encode_json("Ops", usage.ops, formatter);
1314     encode_json("SuccessfulOps", usage.successful_ops, formatter);
1315     formatter->close_section(); // Entry
1316   }
1317   formatter->close_section(); // Category
1318 }
1319
1320 static void dump_usage_bucket_info(Formatter *formatter, const std::string& name, const cls_user_bucket_entry& entry)
1321 {
1322   formatter->open_object_section("Entry");
1323   encode_json("Bucket", name, formatter);
1324   encode_json("Bytes", entry.size, formatter);
1325   encode_json("Bytes_Rounded", entry.size_rounded, formatter);
1326   formatter->close_section(); // entry
1327 }
1328
1329 void RGWGetUsage_ObjStore_S3::send_response()
1330 {
1331   if (op_ret < 0)
1332     set_req_state_err(s, op_ret);
1333   dump_errno(s);
1334
1335   // Explicitly use chunked transfer encoding so that we can stream the result
1336   // to the user without having to wait for the full length of it.
1337   end_header(s, this, "application/xml", CHUNKED_TRANSFER_ENCODING);
1338   dump_start(s);
1339   if (op_ret < 0)
1340     return;
1341
1342   Formatter *formatter = s->formatter;
1343   string last_owner;
1344   bool user_section_open = false;
1345
1346   formatter->open_object_section("Usage");
1347   if (show_log_entries) {
1348     formatter->open_array_section("Entries");
1349   }
1350   map<rgw_user_bucket, rgw_usage_log_entry>::iterator iter;
1351   for (iter = usage.begin(); iter != usage.end(); ++iter) {
1352     const rgw_user_bucket& ub = iter->first;
1353     const rgw_usage_log_entry& entry = iter->second;
1354
1355     if (show_log_entries) {
1356       if (ub.user.compare(last_owner) != 0) {
1357         if (user_section_open) {
1358           formatter->close_section();
1359           formatter->close_section();
1360         }
1361         formatter->open_object_section("User");
1362         formatter->dump_string("Owner", ub.user);
1363         formatter->open_array_section("Buckets");
1364         user_section_open = true;
1365         last_owner = ub.user;
1366       }
1367       formatter->open_object_section("Bucket");
1368       formatter->dump_string("Bucket", ub.bucket);
1369       utime_t ut(entry.epoch, 0);
1370       ut.gmtime(formatter->dump_stream("Time"));
1371       formatter->dump_int("Epoch", entry.epoch);
1372       dump_usage_categories_info(formatter, entry, &categories);
1373       formatter->close_section(); // bucket
1374     }
1375
1376     summary_map[ub.user].aggregate(entry, &categories);
1377   }
1378
1379   if (show_log_entries) {
1380      if (user_section_open) {
1381        formatter->close_section(); // buckets
1382        formatter->close_section(); //user
1383      }
1384      formatter->close_section(); // entries
1385    }
1386
1387    if (show_log_sum) {
1388      formatter->open_array_section("Summary");
1389      map<string, rgw_usage_log_entry>::iterator siter;
1390      for (siter = summary_map.begin(); siter != summary_map.end(); ++siter) {
1391        const rgw_usage_log_entry& entry = siter->second;
1392        formatter->open_object_section("User");
1393        formatter->dump_string("User", siter->first);
1394        dump_usage_categories_info(formatter, entry, &categories);
1395        rgw_usage_data total_usage;
1396        entry.sum(total_usage, categories);
1397        formatter->open_object_section("Total");
1398        encode_json("BytesSent", total_usage.bytes_sent, formatter);
1399        encode_json("BytesReceived", total_usage.bytes_received, formatter);
1400        encode_json("Ops", total_usage.ops, formatter);
1401        encode_json("SuccessfulOps", total_usage.successful_ops, formatter);
1402        formatter->close_section(); // total
1403        formatter->close_section(); // user
1404      }
1405
1406      if (s->cct->_conf->rgw_rest_getusage_op_compat) {
1407        formatter->open_object_section("Stats");
1408      }
1409
1410      encode_json("TotalBytes", stats.size, formatter);
1411      encode_json("TotalBytesRounded", stats.size_rounded, formatter);
1412      encode_json("TotalEntries", stats.num_objects, formatter);
1413
1414      if (s->cct->_conf->rgw_rest_getusage_op_compat) {
1415        formatter->close_section(); //Stats
1416      }
1417
1418      formatter->close_section(); // summary
1419    }
1420
1421   formatter->open_array_section("CapacityUsed");
1422   formatter->open_object_section("User");
1423   formatter->open_array_section("Buckets");
1424   for (const auto& biter : buckets_usage) {
1425     const cls_user_bucket_entry& entry = biter.second;
1426     dump_usage_bucket_info(formatter, biter.first, entry);
1427   }
1428   formatter->close_section(); // Buckets
1429   formatter->close_section(); // User
1430   formatter->close_section(); // CapacityUsed
1431
1432   formatter->close_section(); // usage
1433   rgw_flush_formatter_and_reset(s, s->formatter);
1434 }
1435
1436 int RGWListBucket_ObjStore_S3::get_common_params()
1437 {
1438   list_versions = s->info.args.exists("versions");
1439   prefix = s->info.args.get("prefix");
1440
1441   // non-standard
1442   s->info.args.get_bool("allow-unordered", &allow_unordered, false);
1443   delimiter = s->info.args.get("delimiter");
1444   max_keys = s->info.args.get("max-keys");
1445   op_ret = parse_max_keys();
1446   if (op_ret < 0) {
1447    return op_ret;
1448   }
1449   encoding_type = s->info.args.get("encoding-type");
1450   if (s->system_request) {
1451     s->info.args.get_bool("objs-container", &objs_container, false);
1452     const char *shard_id_str = s->info.env->get("HTTP_RGWX_SHARD_ID");
1453     if (shard_id_str) {
1454       string err;
1455       shard_id = strict_strtol(shard_id_str, 10, &err);
1456       if (!err.empty()) {
1457         ldout(s->cct, 5) << "bad shard id specified: " << shard_id_str << dendl;
1458         return -EINVAL;
1459       }
1460     } else {
1461      shard_id = s->bucket_instance_shard_id;
1462     }
1463   }
1464   return 0;
1465 }
1466
1467 int RGWListBucket_ObjStore_S3::get_params()
1468 {
1469   int ret = get_common_params();
1470   if (ret < 0) {
1471     return ret;
1472   }
1473   if (!list_versions) {
1474     marker = s->info.args.get("marker");
1475   } else {
1476     marker.name = s->info.args.get("key-marker");
1477     marker.instance = s->info.args.get("version-id-marker");
1478   }
1479   return 0;
1480 }
1481
1482 int RGWListBucket_ObjStore_S3v2::get_params()
1483 {
1484 int ret = get_common_params();
1485 if (ret < 0) {
1486   return ret;
1487 }
1488 s->info.args.get_bool("fetch-owner", &fetchOwner, false);
1489 startAfter = s->info.args.get("start-after", &start_after_exist);
1490 continuation_token = s->info.args.get("continuation-token", &continuation_token_exist);
1491 if(!continuation_token_exist) {
1492   marker = startAfter;
1493 } else {
1494   marker = continuation_token;
1495 }
1496 return 0;
1497 }
1498
1499 void RGWListBucket_ObjStore_S3::send_common_versioned_response()
1500 {
1501   if (!s->bucket_tenant.empty()) {
1502     s->formatter->dump_string("Tenant", s->bucket_tenant);
1503   }
1504   s->formatter->dump_string("Name", s->bucket_name);
1505   s->formatter->dump_string("Prefix", prefix);
1506   s->formatter->dump_int("MaxKeys", max);
1507   if (!delimiter.empty()) {
1508     s->formatter->dump_string("Delimiter", delimiter);
1509   }
1510   s->formatter->dump_string("IsTruncated", (max && is_truncated ? "true"
1511               : "false"));
1512
1513   if (!common_prefixes.empty()) {
1514       map<string, bool>::iterator pref_iter;
1515       for (pref_iter = common_prefixes.begin();
1516       pref_iter != common_prefixes.end(); ++pref_iter) {
1517       s->formatter->open_array_section("CommonPrefixes");
1518       if (encode_key) {
1519         s->formatter->dump_string("Prefix", url_encode(pref_iter->first, false));
1520       } else {
1521         s->formatter->dump_string("Prefix", pref_iter->first);
1522       }
1523
1524       s->formatter->close_section();
1525       }
1526     }
1527   }
1528
1529 void RGWListBucket_ObjStore_S3::send_versioned_response()
1530 {
1531   s->formatter->open_object_section_in_ns("ListVersionsResult", XMLNS_AWS_S3);
1532   if (strcasecmp(encoding_type.c_str(), "url") == 0) {
1533     s->formatter->dump_string("EncodingType", "url");
1534     encode_key = true;
1535   }
1536   RGWListBucket_ObjStore_S3::send_common_versioned_response();
1537   s->formatter->dump_string("KeyMarker", marker.name);
1538   s->formatter->dump_string("VersionIdMarker", marker.instance);
1539   if (is_truncated && !next_marker.empty()) {
1540     s->formatter->dump_string("NextKeyMarker", next_marker.name);
1541     if (next_marker.instance.empty()) {
1542       s->formatter->dump_string("NextVersionIdMarker", "null");
1543     }
1544     else {
1545       s->formatter->dump_string("NextVersionIdMarker", next_marker.instance);
1546     }
1547   }
1548
1549   if (op_ret >= 0) {
1550     if (objs_container) {
1551       s->formatter->open_array_section("Entries");
1552     }
1553
1554     vector<rgw_bucket_dir_entry>::iterator iter;
1555     for (iter = objs.begin(); iter != objs.end(); ++iter) {
1556       const char *section_name = (iter->is_delete_marker() ? "DeleteMarker"
1557           : "Version");
1558       s->formatter->open_object_section(section_name);
1559       if (objs_container) {
1560         s->formatter->dump_bool("IsDeleteMarker", iter->is_delete_marker());
1561       }
1562       rgw_obj_key key(iter->key);
1563       if (encode_key) {
1564         string key_name;
1565         url_encode(key.name, key_name);
1566         s->formatter->dump_string("Key", key_name);
1567       }
1568       else {
1569         s->formatter->dump_string("Key", key.name);
1570       }
1571       string version_id = key.instance;
1572       if (version_id.empty()) {
1573         version_id = "null";
1574       }
1575       if (s->system_request) {
1576         if (iter->versioned_epoch > 0) {
1577           s->formatter->dump_int("VersionedEpoch", iter->versioned_epoch);
1578         }
1579         s->formatter->dump_string("RgwxTag", iter->tag);
1580         utime_t ut(iter->meta.mtime);
1581         ut.gmtime_nsec(s->formatter->dump_stream("RgwxMtime"));
1582       }
1583       s->formatter->dump_string("VersionId", version_id);
1584       s->formatter->dump_bool("IsLatest", iter->is_current());
1585       dump_time(s, "LastModified", &iter->meta.mtime);
1586       if (!iter->is_delete_marker()) {
1587         s->formatter->dump_format("ETag", "\"%s\"", iter->meta.etag.c_str());
1588         s->formatter->dump_int("Size", iter->meta.accounted_size);
1589         auto& storage_class = rgw_placement_rule::get_canonical_storage_class(iter->meta.storage_class);
1590         s->formatter->dump_string("StorageClass", storage_class.c_str());
1591       }
1592       dump_owner(s, rgw_user(iter->meta.owner), iter->meta.owner_display_name);
1593       if (iter->meta.appendable) {
1594         s->formatter->dump_string("Type", "Appendable");
1595       } else {
1596         s->formatter->dump_string("Type", "Normal");
1597       }
1598       s->formatter->close_section(); // Version/DeleteMarker
1599     }
1600     if (objs_container) {
1601       s->formatter->close_section(); // Entries
1602     }
1603     s->formatter->close_section(); // ListVersionsResult
1604   }
1605   rgw_flush_formatter_and_reset(s, s->formatter);
1606 }
1607
1608
1609 void RGWListBucket_ObjStore_S3::send_common_response()
1610 {
1611   if (!s->bucket_tenant.empty()) {
1612     s->formatter->dump_string("Tenant", s->bucket_tenant);
1613   }
1614   s->formatter->dump_string("Name", s->bucket_name);
1615   s->formatter->dump_string("Prefix", prefix);
1616   s->formatter->dump_int("MaxKeys", max);
1617   if (!delimiter.empty()) {
1618     s->formatter->dump_string("Delimiter", delimiter);
1619   }
1620   s->formatter->dump_string("IsTruncated", (max && is_truncated ? "true"
1621               : "false"));
1622
1623     if (!common_prefixes.empty()) {
1624       map<string, bool>::iterator pref_iter;
1625       for (pref_iter = common_prefixes.begin();
1626       pref_iter != common_prefixes.end(); ++pref_iter) {
1627       s->formatter->open_array_section("CommonPrefixes");
1628       if (encode_key) {
1629         s->formatter->dump_string("Prefix", url_encode(pref_iter->first, false));
1630       } else {
1631         s->formatter->dump_string("Prefix", pref_iter->first);
1632       }
1633       s->formatter->close_section();
1634       }
1635     }
1636   }
1637
1638 void RGWListBucket_ObjStore_S3::send_response()
1639 {
1640   if (op_ret < 0) {
1641     set_req_state_err(s, op_ret);
1642   }
1643   dump_errno(s);
1644
1645   // Explicitly use chunked transfer encoding so that we can stream the result
1646   // to the user without having to wait for the full length of it.
1647   end_header(s, this, "application/xml", CHUNKED_TRANSFER_ENCODING);
1648   dump_start(s);
1649   if (op_ret < 0) {
1650     return;
1651   }
1652   if (list_versions) {
1653     send_versioned_response();
1654     return;
1655   }
1656
1657   s->formatter->open_object_section_in_ns("ListBucketResult", XMLNS_AWS_S3);
1658   if (strcasecmp(encoding_type.c_str(), "url") == 0) {
1659     s->formatter->dump_string("EncodingType", "url");
1660     encode_key = true;
1661   }
1662   RGWListBucket_ObjStore_S3::send_common_response();
1663     if (op_ret >= 0) {
1664       vector<rgw_bucket_dir_entry>::iterator iter;
1665       for (iter = objs.begin(); iter != objs.end(); ++iter) {
1666         rgw_obj_key key(iter->key);
1667         s->formatter->open_array_section("Contents");
1668         if (encode_key) {
1669           string key_name;
1670           url_encode(key.name, key_name);
1671           s->formatter->dump_string("Key", key_name);
1672       } else {
1673           s->formatter->dump_string("Key", key.name);
1674       }
1675         dump_time(s, "LastModified", &iter->meta.mtime);
1676         s->formatter->dump_format("ETag", "\"%s\"", iter->meta.etag.c_str());
1677         s->formatter->dump_int("Size", iter->meta.accounted_size);
1678         auto& storage_class = rgw_placement_rule::get_canonical_storage_class(iter->meta.storage_class);
1679         s->formatter->dump_string("StorageClass", storage_class.c_str());
1680         dump_owner(s, rgw_user(iter->meta.owner), iter->meta.owner_display_name);
1681         if (s->system_request) {
1682           s->formatter->dump_string("RgwxTag", iter->tag);
1683       }
1684         if (iter->meta.appendable) {
1685           s->formatter->dump_string("Type", "Appendable");
1686       } else {
1687           s->formatter->dump_string("Type", "Normal");
1688       }
1689         s->formatter->close_section();
1690     }
1691   }
1692     s->formatter->dump_string("Marker", marker.name);
1693     if (is_truncated && !next_marker.empty()) {
1694       s->formatter->dump_string("NextMarker", next_marker.name);
1695     }
1696     s->formatter->close_section();
1697     rgw_flush_formatter_and_reset(s, s->formatter);
1698 }
1699
1700 void RGWListBucket_ObjStore_S3v2::send_versioned_response()
1701 {
1702   s->formatter->open_object_section_in_ns("ListVersionsResult", XMLNS_AWS_S3);
1703   RGWListBucket_ObjStore_S3v2::send_common_versioned_response();
1704   s->formatter->dump_string("KeyContinuationToken", marker.name);
1705   s->formatter->dump_string("VersionIdContinuationToken", marker.instance);
1706   if (is_truncated && !next_marker.empty()) {
1707     s->formatter->dump_string("NextKeyContinuationToken", next_marker.name);
1708     s->formatter->dump_string("NextVersionIdContinuationToken", next_marker.instance);
1709   }
1710
1711   if (strcasecmp(encoding_type.c_str(), "url") == 0) {
1712     s->formatter->dump_string("EncodingType", "url");
1713     encode_key = true;
1714   }
1715
1716   if (op_ret >= 0) {
1717     if (objs_container) {
1718       s->formatter->open_array_section("Entries");
1719     }
1720
1721     vector<rgw_bucket_dir_entry>::iterator iter;
1722     for (iter = objs.begin(); iter != objs.end(); ++iter) {
1723       const char *section_name = (iter->is_delete_marker() ? "DeleteContinuationToken"
1724           : "Version");
1725       s->formatter->open_object_section(section_name);
1726       if (objs_container) {
1727         s->formatter->dump_bool("IsDeleteContinuationToken", iter->is_delete_marker());
1728       }
1729       rgw_obj_key key(iter->key);
1730       if (encode_key) {
1731         string key_name;
1732         url_encode(key.name, key_name);
1733         s->formatter->dump_string("Key", key_name);
1734       }
1735       else {
1736         s->formatter->dump_string("Key", key.name);
1737       }
1738       string version_id = key.instance;
1739       if (version_id.empty()) {
1740         version_id = "null";
1741       }
1742       if (s->system_request) {
1743         if (iter->versioned_epoch > 0) {
1744           s->formatter->dump_int("VersionedEpoch", iter->versioned_epoch);
1745         }
1746         s->formatter->dump_string("RgwxTag", iter->tag);
1747         utime_t ut(iter->meta.mtime);
1748         ut.gmtime_nsec(s->formatter->dump_stream("RgwxMtime"));
1749       }
1750       s->formatter->dump_string("VersionId", version_id);
1751       s->formatter->dump_bool("IsLatest", iter->is_current());
1752       dump_time(s, "LastModified", &iter->meta.mtime);
1753       if (!iter->is_delete_marker()) {
1754         s->formatter->dump_format("ETag", "\"%s\"", iter->meta.etag.c_str());
1755         s->formatter->dump_int("Size", iter->meta.accounted_size);
1756         auto& storage_class = rgw_placement_rule::get_canonical_storage_class(iter->meta.storage_class);
1757         s->formatter->dump_string("StorageClass", storage_class.c_str());
1758       }
1759       if (fetchOwner == true) {
1760         dump_owner(s, s->user->get_id(), s->user->get_display_name());
1761       }
1762       s->formatter->close_section();
1763     }
1764
1765
1766     if (objs_container) {
1767       s->formatter->close_section();
1768     }
1769
1770      if (!common_prefixes.empty()) {
1771       map<string, bool>::iterator pref_iter;
1772       for (pref_iter = common_prefixes.begin();
1773       pref_iter != common_prefixes.end(); ++pref_iter) {
1774       s->formatter->open_array_section("CommonPrefixes");
1775       if (encode_key) {
1776         s->formatter->dump_string("Prefix", url_encode(pref_iter->first, false));
1777       } else {
1778         s->formatter->dump_string("Prefix", pref_iter->first);
1779       }
1780
1781       s->formatter->dump_int("KeyCount",objs.size());
1782       if (start_after_exist) {
1783         s->formatter->dump_string("StartAfter", startAfter);
1784       }
1785       s->formatter->close_section();
1786       }
1787     }
1788
1789     s->formatter->close_section();
1790     rgw_flush_formatter_and_reset(s, s->formatter);
1791   }
1792 }
1793
1794 void RGWListBucket_ObjStore_S3v2::send_response()
1795 {
1796   if (op_ret < 0) {
1797     set_req_state_err(s, op_ret);
1798   }
1799   dump_errno(s);
1800
1801   // Explicitly use chunked transfer encoding so that we can stream the result
1802   // to the user without having to wait for the full length of it.
1803   end_header(s, this, "application/xml", CHUNKED_TRANSFER_ENCODING);
1804   dump_start(s);
1805   if (op_ret < 0) {
1806     return;
1807   }
1808   if (list_versions) {
1809     send_versioned_response();
1810     return;
1811   }
1812
1813   s->formatter->open_object_section_in_ns("ListBucketResult", XMLNS_AWS_S3);
1814   if (strcasecmp(encoding_type.c_str(), "url") == 0) {
1815     s->formatter->dump_string("EncodingType", "url");
1816     encode_key = true;
1817   }
1818
1819   RGWListBucket_ObjStore_S3::send_common_response();
1820   if (op_ret >= 0) {
1821     vector<rgw_bucket_dir_entry>::iterator iter;
1822     for (iter = objs.begin(); iter != objs.end(); ++iter) {
1823       rgw_obj_key key(iter->key);
1824       s->formatter->open_array_section("Contents");
1825       if (encode_key) {
1826         string key_name;
1827         url_encode(key.name, key_name);
1828         s->formatter->dump_string("Key", key_name);
1829       }
1830       else {
1831         s->formatter->dump_string("Key", key.name);
1832       }
1833       dump_time(s, "LastModified", &iter->meta.mtime);
1834       s->formatter->dump_format("ETag", "\"%s\"", iter->meta.etag.c_str());
1835       s->formatter->dump_int("Size", iter->meta.accounted_size);
1836       auto& storage_class = rgw_placement_rule::get_canonical_storage_class(iter->meta.storage_class);
1837       s->formatter->dump_string("StorageClass", storage_class.c_str());
1838       if (fetchOwner == true) {
1839         dump_owner(s, s->user->get_id(), s->user->get_display_name());
1840       }
1841       if (s->system_request) {
1842         s->formatter->dump_string("RgwxTag", iter->tag);
1843       }
1844       if (iter->meta.appendable) {
1845         s->formatter->dump_string("Type", "Appendable");
1846       } else {
1847         s->formatter->dump_string("Type", "Normal");
1848       }
1849       s->formatter->close_section();
1850     }
1851   }
1852   if (continuation_token_exist) {
1853     s->formatter->dump_string("ContinuationToken", continuation_token);
1854   }
1855   if (is_truncated && !next_marker.empty()) {
1856     s->formatter->dump_string("NextContinuationToken", next_marker.name);
1857   }
1858   s->formatter->dump_int("KeyCount", objs.size() + common_prefixes.size());
1859   if (start_after_exist) {
1860     s->formatter->dump_string("StartAfter", startAfter);
1861   }
1862   s->formatter->close_section();
1863   rgw_flush_formatter_and_reset(s, s->formatter);
1864 }
1865
1866 void RGWGetBucketLogging_ObjStore_S3::send_response()
1867 {
1868   dump_errno(s);
1869   end_header(s, this, "application/xml");
1870   dump_start(s);
1871
1872   s->formatter->open_object_section_in_ns("BucketLoggingStatus", XMLNS_AWS_S3);
1873   s->formatter->close_section();
1874   rgw_flush_formatter_and_reset(s, s->formatter);
1875 }
1876
1877 void RGWGetBucketLocation_ObjStore_S3::send_response()
1878 {
1879   dump_errno(s);
1880   end_header(s, this);
1881   dump_start(s);
1882
1883   RGWZoneGroup zonegroup;
1884   string api_name;
1885
1886   int ret = store->svc()->zone->get_zonegroup(s->bucket_info.zonegroup, zonegroup);
1887   if (ret >= 0) {
1888     api_name = zonegroup.api_name;
1889   } else  {
1890     if (s->bucket_info.zonegroup != "default") {
1891       api_name = s->bucket_info.zonegroup;
1892     }
1893   }
1894
1895   s->formatter->dump_format_ns("LocationConstraint", XMLNS_AWS_S3,
1896                                "%s", api_name.c_str());
1897   rgw_flush_formatter_and_reset(s, s->formatter);
1898 }
1899
1900 void RGWGetBucketVersioning_ObjStore_S3::send_response()
1901 {
1902   dump_errno(s);
1903   end_header(s, this, "application/xml");
1904   dump_start(s);
1905
1906   s->formatter->open_object_section_in_ns("VersioningConfiguration", XMLNS_AWS_S3);
1907   if (versioned) {
1908     const char *status = (versioning_enabled ? "Enabled" : "Suspended");
1909     s->formatter->dump_string("Status", status);
1910     const char *mfa_status = (mfa_enabled ? "Enabled" : "Disabled");
1911     s->formatter->dump_string("MfaDelete", mfa_status);
1912   }
1913   s->formatter->close_section();
1914   rgw_flush_formatter_and_reset(s, s->formatter);
1915 }
1916
1917 struct ver_config_status {
1918   int status{VersioningSuspended};
1919
1920   enum MFAStatus {
1921     MFA_UNKNOWN,
1922     MFA_DISABLED,
1923     MFA_ENABLED,
1924   } mfa_status{MFA_UNKNOWN};
1925   int retcode{0};
1926
1927   void decode_xml(XMLObj *obj) {
1928     string status_str;
1929     string mfa_str;
1930     RGWXMLDecoder::decode_xml("Status", status_str, obj);
1931     if (status_str == "Enabled") {
1932       status = VersioningEnabled;
1933     } else if (status_str != "Suspended") {
1934       status = VersioningStatusInvalid;
1935     }
1936
1937
1938     if (RGWXMLDecoder::decode_xml("MfaDelete", mfa_str, obj)) {
1939       if (mfa_str == "Enabled") {
1940         mfa_status = MFA_ENABLED;
1941       } else if (mfa_str == "Disabled") {
1942         mfa_status = MFA_DISABLED;
1943       } else {
1944         retcode = -EINVAL;
1945       }
1946     }
1947   }
1948 };
1949
1950 int RGWSetBucketVersioning_ObjStore_S3::get_params()
1951 {
1952   int r = 0;
1953   bufferlist data;
1954   std::tie(r, data) =
1955     rgw_rest_read_all_input(s, s->cct->_conf->rgw_max_put_param_size, false);
1956   if (r < 0) {
1957     return r;
1958   }
1959
1960   r = do_aws4_auth_completion();
1961   if (r < 0) {
1962     return r;
1963   }
1964
1965   RGWXMLDecoder::XMLParser parser;
1966   if (!parser.init()) {
1967     ldpp_dout(this, 0) << "ERROR: failed to initialize parser" << dendl;
1968     return -EIO;
1969   }
1970
1971   char* buf = data.c_str();
1972   if (!parser.parse(buf, data.length(), 1)) {
1973     ldpp_dout(this, 10) << "NOTICE: failed to parse data: " << buf << dendl;
1974     r = -EINVAL;
1975     return r;
1976   }
1977
1978   ver_config_status status_conf;
1979
1980   if (!RGWXMLDecoder::decode_xml("VersioningConfiguration", status_conf, &parser)) {
1981     ldpp_dout(this, 10) << "NOTICE: bad versioning config input" << dendl;
1982     return -EINVAL;
1983   }
1984
1985   if (!store->svc()->zone->is_meta_master()) {
1986     /* only need to keep this data around if we're not meta master */
1987     in_data.append(data);
1988   }
1989
1990   versioning_status = status_conf.status;
1991   if (versioning_status == VersioningStatusInvalid) {
1992     r = -EINVAL;
1993   }
1994
1995   if (status_conf.mfa_status != ver_config_status::MFA_UNKNOWN) {
1996     mfa_set_status = true;
1997     switch (status_conf.mfa_status) {
1998       case ver_config_status::MFA_DISABLED:
1999         mfa_status = false;
2000         break;
2001       case ver_config_status::MFA_ENABLED:
2002         mfa_status = true;
2003         break;
2004       default:
2005         ldpp_dout(this, 0) << "ERROR: RGWSetBucketVersioning_ObjStore_S3::get_params(): unexpected switch case mfa_status=" << status_conf.mfa_status << dendl;
2006         r = -EIO;
2007     }
2008   } else if (status_conf.retcode < 0) {
2009     r = status_conf.retcode;
2010   }
2011   return r;
2012 }
2013
2014 void RGWSetBucketVersioning_ObjStore_S3::send_response()
2015 {
2016   if (op_ret)
2017     set_req_state_err(s, op_ret);
2018   dump_errno(s);
2019   end_header(s, this, "application/xml");
2020 }
2021
2022 int RGWSetBucketWebsite_ObjStore_S3::get_params()
2023 {
2024   const auto max_size = s->cct->_conf->rgw_max_put_param_size;
2025
2026   int r = 0;
2027   bufferlist data;
2028   std::tie(r, data) = rgw_rest_read_all_input(s, max_size, false);
2029
2030   if (r < 0) {
2031     return r;
2032   }
2033
2034   r = do_aws4_auth_completion();
2035   if (r < 0) {
2036     return r;
2037   }
2038
2039   in_data.append(data);
2040
2041   RGWXMLDecoder::XMLParser parser;
2042   if (!parser.init()) {
2043     ldpp_dout(this, 0) << "ERROR: failed to initialize parser" << dendl;
2044     return -EIO;
2045   }
2046
2047   char* buf = data.c_str();
2048   if (!parser.parse(buf, data.length(), 1)) {
2049     ldpp_dout(this, 5) << "failed to parse xml: " << buf << dendl;
2050     return -EINVAL;
2051   }
2052
2053   try {
2054     RGWXMLDecoder::decode_xml("WebsiteConfiguration", website_conf, &parser, true);
2055   } catch (RGWXMLDecoder::err& err) {
2056     ldpp_dout(this, 5) << "unexpected xml: " << buf << dendl;
2057     return -EINVAL;
2058   }
2059
2060   if (website_conf.is_redirect_all && website_conf.redirect_all.hostname.empty()) {
2061     s->err.message = "A host name must be provided to redirect all requests (e.g. \"example.com\").";
2062     ldout(s->cct, 5) << s->err.message << dendl;
2063     return -EINVAL;
2064   } else if (!website_conf.is_redirect_all && !website_conf.is_set_index_doc) {
2065     s->err.message = "A value for IndexDocument Suffix must be provided if RedirectAllRequestsTo is empty";
2066     ldout(s->cct, 5) << s->err.message << dendl;
2067     return -EINVAL;
2068   } else if (!website_conf.is_redirect_all && website_conf.is_set_index_doc &&
2069              website_conf.index_doc_suffix.empty()) {
2070     s->err.message = "The IndexDocument Suffix is not well formed";
2071     ldout(s->cct, 5) << s->err.message << dendl;
2072     return -EINVAL;
2073   }
2074
2075 #define WEBSITE_ROUTING_RULES_MAX_NUM      50
2076   int max_num = s->cct->_conf->rgw_website_routing_rules_max_num;
2077   if (max_num < 0) {
2078     max_num = WEBSITE_ROUTING_RULES_MAX_NUM;
2079   }
2080   int routing_rules_num = website_conf.routing_rules.rules.size();
2081   if (routing_rules_num > max_num) {
2082     ldpp_dout(this, 4) << "An website routing config can have up to "
2083                      << max_num
2084                      << " rules, request website routing rules num: "
2085                      << routing_rules_num << dendl;
2086     op_ret = -ERR_INVALID_WEBSITE_ROUTING_RULES_ERROR;
2087     s->err.message = std::to_string(routing_rules_num) +" routing rules provided, the number of routing rules in a website configuration is limited to "
2088                      + std::to_string(max_num)
2089                      + ".";
2090     return -ERR_INVALID_REQUEST;
2091   }
2092
2093   return 0;
2094 }
2095
2096 void RGWSetBucketWebsite_ObjStore_S3::send_response()
2097 {
2098   if (op_ret < 0)
2099     set_req_state_err(s, op_ret);
2100   dump_errno(s);
2101   end_header(s, this, "application/xml");
2102 }
2103
2104 void RGWDeleteBucketWebsite_ObjStore_S3::send_response()
2105 {
2106   if (op_ret == 0) {
2107     op_ret = STATUS_NO_CONTENT;
2108   }
2109   set_req_state_err(s, op_ret);
2110   dump_errno(s);
2111   end_header(s, this, "application/xml");
2112 }
2113
2114 void RGWGetBucketWebsite_ObjStore_S3::send_response()
2115 {
2116   if (op_ret)
2117     set_req_state_err(s, op_ret);
2118   dump_errno(s);
2119   end_header(s, this, "application/xml");
2120   dump_start(s);
2121
2122   if (op_ret < 0) {
2123     return;
2124   }
2125
2126   RGWBucketWebsiteConf& conf = s->bucket_info.website_conf;
2127
2128   s->formatter->open_object_section_in_ns("WebsiteConfiguration", XMLNS_AWS_S3);
2129   conf.dump_xml(s->formatter);
2130   s->formatter->close_section(); // WebsiteConfiguration
2131   rgw_flush_formatter_and_reset(s, s->formatter);
2132 }
2133
2134 static void dump_bucket_metadata(struct req_state *s, rgw::sal::RGWBucket* bucket)
2135 {
2136   dump_header(s, "X-RGW-Object-Count", static_cast<long long>(bucket->get_count()));
2137   dump_header(s, "X-RGW-Bytes-Used", static_cast<long long>(bucket->get_size()));
2138 }
2139
2140 void RGWStatBucket_ObjStore_S3::send_response()
2141 {
2142   if (op_ret >= 0) {
2143     dump_bucket_metadata(s, bucket);
2144   }
2145
2146   set_req_state_err(s, op_ret);
2147   dump_errno(s);
2148
2149   end_header(s, this);
2150   dump_start(s);
2151 }
2152
2153 static int create_s3_policy(struct req_state *s, rgw::sal::RGWRadosStore *store,
2154                             RGWAccessControlPolicy_S3& s3policy,
2155                             ACLOwner& owner)
2156 {
2157   if (s->has_acl_header) {
2158     if (!s->canned_acl.empty())
2159       return -ERR_INVALID_REQUEST;
2160
2161     return s3policy.create_from_headers(store->ctl()->user, s->info.env, owner);
2162   }
2163
2164   return s3policy.create_canned(owner, s->bucket_owner, s->canned_acl);
2165 }
2166
2167 class RGWLocationConstraint : public XMLObj
2168 {
2169 public:
2170   RGWLocationConstraint() {}
2171   ~RGWLocationConstraint() override {}
2172   bool xml_end(const char *el) override {
2173     if (!el)
2174       return false;
2175
2176     location_constraint = get_data();
2177
2178     return true;
2179   }
2180
2181   string location_constraint;
2182 };
2183
2184 class RGWCreateBucketConfig : public XMLObj
2185 {
2186 public:
2187   RGWCreateBucketConfig() {}
2188   ~RGWCreateBucketConfig() override {}
2189 };
2190
2191 class RGWCreateBucketParser : public RGWXMLParser
2192 {
2193   XMLObj *alloc_obj(const char *el) override {
2194     return new XMLObj;
2195   }
2196
2197 public:
2198   RGWCreateBucketParser() {}
2199   ~RGWCreateBucketParser() override {}
2200
2201   bool get_location_constraint(string& zone_group) {
2202     XMLObj *config = find_first("CreateBucketConfiguration");
2203     if (!config)
2204       return false;
2205
2206     XMLObj *constraint = config->find_first("LocationConstraint");
2207     if (!constraint)
2208       return false;
2209
2210     zone_group = constraint->get_data();
2211
2212     return true;
2213   }
2214 };
2215
2216 int RGWCreateBucket_ObjStore_S3::get_params()
2217 {
2218   RGWAccessControlPolicy_S3 s3policy(s->cct);
2219   bool relaxed_names = s->cct->_conf->rgw_relaxed_s3_bucket_names;
2220
2221   int r = valid_s3_bucket_name(s->bucket_name, relaxed_names);
2222   if (r)
2223     return r;
2224
2225   r = create_s3_policy(s, store, s3policy, s->owner);
2226   if (r < 0)
2227     return r;
2228
2229   policy = s3policy;
2230
2231   const auto max_size = s->cct->_conf->rgw_max_put_param_size;
2232
2233   int op_ret = 0;
2234   bufferlist data;
2235   std::tie(op_ret, data) = rgw_rest_read_all_input(s, max_size, false);
2236
2237   if ((op_ret < 0) && (op_ret != -ERR_LENGTH_REQUIRED))
2238     return op_ret;
2239
2240   const int auth_ret = do_aws4_auth_completion();
2241   if (auth_ret < 0) {
2242     return auth_ret;
2243   }
2244
2245   in_data.append(data);
2246
2247   if (data.length()) {
2248     RGWCreateBucketParser parser;
2249
2250     if (!parser.init()) {
2251       ldpp_dout(this, 0) << "ERROR: failed to initialize parser" << dendl;
2252       return -EIO;
2253     }
2254
2255     char* buf = data.c_str();
2256     bool success = parser.parse(buf, data.length(), 1);
2257     ldpp_dout(this, 20) << "create bucket input data=" << buf << dendl;
2258
2259     if (!success) {
2260       ldpp_dout(this, 0) << "failed to parse input: " << buf << dendl;
2261       return -EINVAL;
2262     }
2263
2264     if (!parser.get_location_constraint(location_constraint)) {
2265       ldpp_dout(this, 0) << "provided input did not specify location constraint correctly" << dendl;
2266       return -EINVAL;
2267     }
2268
2269     ldpp_dout(this, 10) << "create bucket location constraint: "
2270                       << location_constraint << dendl;
2271   }
2272
2273   size_t pos = location_constraint.find(':');
2274   if (pos != string::npos) {
2275     placement_rule.init(location_constraint.substr(pos + 1), s->info.storage_class);
2276     location_constraint = location_constraint.substr(0, pos);
2277   } else {
2278     placement_rule.storage_class = s->info.storage_class;
2279   }
2280   auto iter = s->info.x_meta_map.find("x-amz-bucket-object-lock-enabled");
2281   if (iter != s->info.x_meta_map.end()) {
2282     if (!boost::algorithm::iequals(iter->second, "true") && !boost::algorithm::iequals(iter->second, "false")) {
2283       return -EINVAL;
2284     }
2285     obj_lock_enabled = boost::algorithm::iequals(iter->second, "true");
2286   }
2287   return 0;
2288 }
2289
2290 void RGWCreateBucket_ObjStore_S3::send_response()
2291 {
2292   if (op_ret == -ERR_BUCKET_EXISTS)
2293     op_ret = 0;
2294   if (op_ret)
2295     set_req_state_err(s, op_ret);
2296   dump_errno(s);
2297   end_header(s);
2298
2299   if (op_ret < 0)
2300     return;
2301
2302   if (s->system_request) {
2303     JSONFormatter f; /* use json formatter for system requests output */
2304
2305     f.open_object_section("info");
2306     encode_json("entry_point_object_ver", ep_objv, &f);
2307     encode_json("object_ver", info.objv_tracker.read_version, &f);
2308     encode_json("bucket_info", info, &f);
2309     f.close_section();
2310     rgw_flush_formatter_and_reset(s, &f);
2311   }
2312 }
2313
2314 void RGWDeleteBucket_ObjStore_S3::send_response()
2315 {
2316   int r = op_ret;
2317   if (!r)
2318     r = STATUS_NO_CONTENT;
2319
2320   set_req_state_err(s, r);
2321   dump_errno(s);
2322   end_header(s, this);
2323 }
2324
2325 static inline void map_qs_metadata(struct req_state* s)
2326 {
2327   /* merge S3 valid user metadata from the query-string into
2328    * x_meta_map, which maps them to attributes */
2329   const auto& params = const_cast<RGWHTTPArgs&>(s->info.args).get_params();
2330   for (const auto& elt : params) {
2331     std::string k = boost::algorithm::to_lower_copy(elt.first);
2332     if (k.find("x-amz-meta-") == /* offset */ 0) {
2333       rgw_add_amz_meta_header(s->info.x_meta_map, k, elt.second);
2334     }
2335   }
2336 }
2337
2338 int RGWPutObj_ObjStore_S3::get_params()
2339 {
2340   if (!s->length)
2341     return -ERR_LENGTH_REQUIRED;
2342
2343   map<string, bufferlist> src_attrs;
2344   size_t pos;
2345   int ret;
2346
2347   map_qs_metadata(s);
2348
2349   RGWAccessControlPolicy_S3 s3policy(s->cct);
2350   ret = create_s3_policy(s, store, s3policy, s->owner);
2351   if (ret < 0)
2352     return ret;
2353
2354   policy = s3policy;
2355
2356   if_match = s->info.env->get("HTTP_IF_MATCH");
2357   if_nomatch = s->info.env->get("HTTP_IF_NONE_MATCH");
2358   copy_source = url_decode(s->info.env->get("HTTP_X_AMZ_COPY_SOURCE", ""));
2359   copy_source_range = s->info.env->get("HTTP_X_AMZ_COPY_SOURCE_RANGE");
2360
2361   /* handle x-amz-copy-source */
2362   boost::string_view cs_view(copy_source);
2363   if (! cs_view.empty()) {
2364     if (cs_view[0] == '/')
2365       cs_view.remove_prefix(1);
2366     copy_source_bucket_name = cs_view.to_string();
2367     pos = copy_source_bucket_name.find("/");
2368     if (pos == std::string::npos) {
2369       ret = -EINVAL;
2370       ldpp_dout(this, 5) << "x-amz-copy-source bad format" << dendl;
2371       return ret;
2372     }
2373     copy_source_object_name =
2374       copy_source_bucket_name.substr(pos + 1, copy_source_bucket_name.size());
2375     copy_source_bucket_name = copy_source_bucket_name.substr(0, pos);
2376 #define VERSION_ID_STR "?versionId="
2377     pos = copy_source_object_name.find(VERSION_ID_STR);
2378     if (pos == std::string::npos) {
2379       copy_source_object_name = url_decode(copy_source_object_name);
2380     } else {
2381       copy_source_version_id =
2382         copy_source_object_name.substr(pos + sizeof(VERSION_ID_STR) - 1);
2383       copy_source_object_name =
2384         url_decode(copy_source_object_name.substr(0, pos));
2385     }
2386     pos = copy_source_bucket_name.find(":");
2387     if (pos == std::string::npos) {
2388        copy_source_tenant_name = s->src_tenant_name;
2389     } else {
2390        copy_source_tenant_name = copy_source_bucket_name.substr(0, pos);
2391        copy_source_bucket_name = copy_source_bucket_name.substr(pos + 1, copy_source_bucket_name.size());
2392        if (copy_source_bucket_name.empty()) {
2393          ret = -EINVAL;
2394          ldpp_dout(this, 5) << "source bucket name is empty" << dendl;
2395          return ret;
2396        }
2397     }
2398     ret = store->getRados()->get_bucket_info(store->svc(),
2399                                  copy_source_tenant_name,
2400                                  copy_source_bucket_name,
2401                                  copy_source_bucket_info,
2402                                  NULL, s->yield, &src_attrs);
2403     if (ret < 0) {
2404       ldpp_dout(this, 5) << __func__ << "(): get_bucket_info() returned ret=" << ret << dendl;
2405       return ret;
2406     }
2407
2408     /* handle x-amz-copy-source-range */
2409
2410     if (copy_source_range) {
2411       string range = copy_source_range;
2412       pos = range.find("bytes=");
2413       if (pos == std::string::npos || pos != 0) {
2414         ret = -EINVAL;
2415         ldpp_dout(this, 5) << "x-amz-copy-source-range bad format" << dendl;
2416         return ret;
2417       }
2418       /* 6 is the length of "bytes=" */
2419       range = range.substr(pos + 6);
2420       pos = range.find("-");
2421       if (pos == std::string::npos) {
2422         ret = -EINVAL;
2423         ldpp_dout(this, 5) << "x-amz-copy-source-range bad format" << dendl;
2424         return ret;
2425       }
2426       string first = range.substr(0, pos);
2427       string last = range.substr(pos + 1);
2428       if (first.find_first_not_of("0123456789") != std::string::npos || last.find_first_not_of("0123456789") != std::string::npos)
2429       {
2430         ldpp_dout(this, 5) << "x-amz-copy-source-range bad format not an integer" << dendl;
2431         ret = -EINVAL;
2432         return ret;
2433       }
2434       copy_source_range_fst = strtoull(first.c_str(), NULL, 10);
2435       copy_source_range_lst = strtoull(last.c_str(), NULL, 10);
2436       if (copy_source_range_fst > copy_source_range_lst)
2437       {
2438         ret = -ERANGE;
2439         ldpp_dout(this, 5) << "x-amz-copy-source-range bad format first number bigger than second" << dendl;
2440         return ret;
2441       }
2442     }
2443
2444   } /* copy_source */
2445
2446   /* handle object tagging */
2447   auto tag_str = s->info.env->get("HTTP_X_AMZ_TAGGING");
2448   if (tag_str){
2449     obj_tags = std::make_unique<RGWObjTags>();
2450     ret = obj_tags->set_from_string(tag_str);
2451     if (ret < 0){
2452       ldpp_dout(this,0) << "setting obj tags failed with " << ret << dendl;
2453       if (ret == -ERR_INVALID_TAG){
2454         ret = -EINVAL; //s3 returns only -EINVAL for PUT requests
2455       }
2456
2457       return ret;
2458     }
2459   }
2460
2461   //handle object lock
2462   auto obj_lock_mode_str = s->info.env->get("HTTP_X_AMZ_OBJECT_LOCK_MODE");
2463   auto obj_lock_date_str = s->info.env->get("HTTP_X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE");
2464   auto obj_legal_hold_str = s->info.env->get("HTTP_X_AMZ_OBJECT_LOCK_LEGAL_HOLD");
2465   if (obj_lock_mode_str && obj_lock_date_str) {
2466     boost::optional<ceph::real_time> date = ceph::from_iso_8601(obj_lock_date_str);
2467     if (boost::none == date || ceph::real_clock::to_time_t(*date) <= ceph_clock_now()) {
2468         ret = -EINVAL;
2469         ldpp_dout(this,0) << "invalid x-amz-object-lock-retain-until-date value" << dendl;
2470         return ret;
2471     }
2472     if (strcmp(obj_lock_mode_str, "GOVERNANCE") != 0 && strcmp(obj_lock_mode_str, "COMPLIANCE") != 0) {
2473         ret = -EINVAL;
2474         ldpp_dout(this,0) << "invalid x-amz-object-lock-mode value" << dendl;
2475         return ret;
2476     }
2477     obj_retention = new RGWObjectRetention(obj_lock_mode_str, *date);
2478   } else if ((obj_lock_mode_str && !obj_lock_date_str) || (!obj_lock_mode_str && obj_lock_date_str)) {
2479     ret = -EINVAL;
2480     ldpp_dout(this,0) << "need both x-amz-object-lock-mode and x-amz-object-lock-retain-until-date " << dendl;
2481     return ret;
2482   }
2483   if (obj_legal_hold_str) {
2484     if (strcmp(obj_legal_hold_str, "ON") != 0 && strcmp(obj_legal_hold_str, "OFF") != 0) {
2485         ret = -EINVAL;
2486         ldpp_dout(this,0) << "invalid x-amz-object-lock-legal-hold value" << dendl;
2487         return ret;
2488     }
2489     obj_legal_hold = new RGWObjectLegalHold(obj_legal_hold_str);
2490   }
2491   if (!s->bucket_info.obj_lock_enabled() && (obj_retention || obj_legal_hold)) {
2492     ldpp_dout(this, 0) << "ERROR: object retention or legal hold can't be set if bucket object lock not configured" << dendl;
2493     ret = -ERR_INVALID_REQUEST;
2494     return ret;
2495   }
2496   multipart_upload_id = s->info.args.get("uploadId");
2497   multipart_part_str = s->info.args.get("partNumber");
2498   if (!multipart_part_str.empty()) {
2499     string err;
2500     multipart_part_num = strict_strtol(multipart_part_str.c_str(), 10, &err);
2501     if (!err.empty()) {
2502       ldpp_dout(s, 10) << "bad part number: " << multipart_part_str << ": " << err << dendl;
2503       return -EINVAL;
2504     }
2505   } else if (!multipart_upload_id.empty()) {
2506     ldpp_dout(s, 10) << "part number with no multipart upload id" << dendl;
2507     return -EINVAL;
2508   }
2509
2510   append = s->info.args.exists("append");
2511   if (append) {
2512     string pos_str = s->info.args.get("position");
2513     if (pos_str.empty()) {
2514       return -EINVAL;
2515     } else {
2516       position = strtoull(pos_str.c_str(), NULL, 10);
2517     }
2518   }
2519
2520   return RGWPutObj_ObjStore::get_params();
2521 }
2522
2523 int RGWPutObj_ObjStore_S3::get_data(bufferlist& bl)
2524 {
2525   const int ret = RGWPutObj_ObjStore::get_data(bl);
2526   if (ret == 0) {
2527     const int ret_auth = do_aws4_auth_completion();
2528     if (ret_auth < 0) {
2529       return ret_auth;
2530     }
2531   }
2532
2533   return ret;
2534 }
2535
2536 static int get_success_retcode(int code)
2537 {
2538   switch (code) {
2539     case 201:
2540       return STATUS_CREATED;
2541     case 204:
2542       return STATUS_NO_CONTENT;
2543   }
2544   return 0;
2545 }
2546
2547 void RGWPutObj_ObjStore_S3::send_response()
2548 {
2549   if (op_ret) {
2550     set_req_state_err(s, op_ret);
2551     dump_errno(s);
2552   } else {
2553     if (s->cct->_conf->rgw_s3_success_create_obj_status) {
2554       op_ret = get_success_retcode(
2555         s->cct->_conf->rgw_s3_success_create_obj_status);
2556       set_req_state_err(s, op_ret);
2557     }
2558
2559     string expires = get_s3_expiration_header(s, mtime);
2560
2561     if (copy_source.empty()) {
2562       dump_errno(s);
2563       dump_etag(s, etag);
2564       dump_content_length(s, 0);
2565       dump_header_if_nonempty(s, "x-amz-version-id", version_id);
2566       dump_header_if_nonempty(s, "x-amz-expiration", expires);
2567       for (auto &it : crypt_http_responses)
2568         dump_header(s, it.first, it.second);
2569     } else {
2570       dump_errno(s);
2571       dump_header_if_nonempty(s, "x-amz-version-id", version_id);
2572       dump_header_if_nonempty(s, "x-amz-expiration", expires);
2573       end_header(s, this, "application/xml");
2574       dump_start(s);
2575       struct tm tmp;
2576       utime_t ut(mtime);
2577       time_t secs = (time_t)ut.sec();
2578       gmtime_r(&secs, &tmp);
2579       char buf[TIME_BUF_SIZE];
2580       s->formatter->open_object_section_in_ns("CopyPartResult",
2581           "http://s3.amazonaws.com/doc/2006-03-01/");
2582       if (strftime(buf, sizeof(buf), "%Y-%m-%dT%T.000Z", &tmp) > 0) {
2583         s->formatter->dump_string("LastModified", buf);
2584       }
2585       s->formatter->dump_string("ETag", etag);
2586       s->formatter->close_section();
2587       rgw_flush_formatter_and_reset(s, s->formatter);
2588       return;
2589     }
2590   }
2591   if (append) {
2592     if (op_ret == 0 || op_ret == -ERR_POSITION_NOT_EQUAL_TO_LENGTH) {
2593       dump_header(s, "x-rgw-next-append-position", cur_accounted_size);
2594     }
2595   }
2596   if (s->system_request && !real_clock::is_zero(mtime)) {
2597     dump_epoch_header(s, "Rgwx-Mtime", mtime);
2598   }
2599   end_header(s, this);
2600 }
2601
2602 static inline int get_obj_attrs(rgw::sal::RGWRadosStore *store, struct req_state *s, rgw_obj& obj, map<string, bufferlist>& attrs)
2603 {
2604   RGWRados::Object op_target(store->getRados(), s->bucket_info, *static_cast<RGWObjectCtx *>(s->obj_ctx), obj);
2605   RGWRados::Object::Read read_op(&op_target);
2606
2607   read_op.params.attrs = &attrs;
2608
2609   return read_op.prepare(s->yield);
2610 }
2611
2612 static inline void set_attr(map<string, bufferlist>& attrs, const char* key, const std::string& value)
2613 {
2614   bufferlist bl;
2615   encode(value,bl);
2616   attrs.emplace(key, std::move(bl));
2617 }
2618
2619 static inline void set_attr(map<string, bufferlist>& attrs, const char* key, const char* value)
2620 {
2621   bufferlist bl;
2622   encode(value,bl);
2623   attrs.emplace(key, std::move(bl));
2624 }
2625
2626 int RGWPutObj_ObjStore_S3::get_decrypt_filter(
2627     std::unique_ptr<RGWGetObj_Filter>* filter,
2628     RGWGetObj_Filter* cb,
2629     map<string, bufferlist>& attrs,
2630     bufferlist* manifest_bl)
2631 {
2632   std::map<std::string, std::string> crypt_http_responses_unused;
2633
2634   int res = 0;
2635   std::unique_ptr<BlockCrypt> block_crypt;
2636   res = rgw_s3_prepare_decrypt(s, attrs, &block_crypt, crypt_http_responses_unused);
2637   if (res == 0) {
2638     if (block_crypt != nullptr) {
2639       auto f = std::unique_ptr<RGWGetObj_BlockDecrypt>(new RGWGetObj_BlockDecrypt(s->cct, cb, std::move(block_crypt)));
2640       //RGWGetObj_BlockDecrypt* f = new RGWGetObj_BlockDecrypt(s->cct, cb, std::move(block_crypt));
2641       if (f != nullptr) {
2642         if (manifest_bl != nullptr) {
2643           res = f->read_manifest(*manifest_bl);
2644           if (res == 0) {
2645             *filter = std::move(f);
2646           }
2647         }
2648       }
2649     }
2650   }
2651   return res;
2652 }
2653
2654 int RGWPutObj_ObjStore_S3::get_encrypt_filter(
2655     std::unique_ptr<rgw::putobj::DataProcessor> *filter,
2656     rgw::putobj::DataProcessor *cb)
2657 {
2658   int res = 0;
2659   if (!multipart_upload_id.empty()) {
2660     RGWMPObj mp(s->object.name, multipart_upload_id);
2661     rgw_obj obj;
2662     obj.init_ns(s->bucket, mp.get_meta(), RGW_OBJ_NS_MULTIPART);
2663     obj.set_in_extra_data(true);
2664     map<string, bufferlist> xattrs;
2665     res = get_obj_attrs(store, s, obj, xattrs);
2666     if (res == 0) {
2667       std::unique_ptr<BlockCrypt> block_crypt;
2668       /* We are adding to existing object.
2669        * We use crypto mode that configured as if we were decrypting. */
2670       res = rgw_s3_prepare_decrypt(s, xattrs, &block_crypt, crypt_http_responses);
2671       if (res == 0 && block_crypt != nullptr)
2672         filter->reset(new RGWPutObj_BlockEncrypt(s->cct, cb, std::move(block_crypt)));
2673     }
2674     /* it is ok, to not have encryption at all */
2675   }
2676   else
2677   {
2678     std::unique_ptr<BlockCrypt> block_crypt;
2679     res = rgw_s3_prepare_encrypt(s, attrs, nullptr, &block_crypt, crypt_http_responses);
2680     if (res == 0 && block_crypt != nullptr) {
2681       filter->reset(new RGWPutObj_BlockEncrypt(s->cct, cb, std::move(block_crypt)));
2682     }
2683   }
2684   return res;
2685 }
2686
2687 void RGWPostObj_ObjStore_S3::rebuild_key(string& key)
2688 {
2689   static string var = "${filename}";
2690   int pos = key.find(var);
2691   if (pos < 0)
2692     return;
2693
2694   string new_key = key.substr(0, pos);
2695   new_key.append(filename);
2696   new_key.append(key.substr(pos + var.size()));
2697
2698   key = new_key;
2699 }
2700
2701 std::string RGWPostObj_ObjStore_S3::get_current_filename() const
2702 {
2703   return s->object.name;
2704 }
2705
2706 std::string RGWPostObj_ObjStore_S3::get_current_content_type() const
2707 {
2708   return content_type;
2709 }
2710
2711 int RGWPostObj_ObjStore_S3::get_params()
2712 {
2713   op_ret = RGWPostObj_ObjStore::get_params();
2714   if (op_ret < 0) {
2715     return op_ret;
2716   }
2717
2718   map_qs_metadata(s);
2719
2720   ldpp_dout(this, 20) << "adding bucket to policy env: " << s->bucket.name
2721                     << dendl;
2722   env.add_var("bucket", s->bucket.name);
2723
2724   bool done;
2725   do {
2726     struct post_form_part part;
2727     int r = read_form_part_header(&part, done);
2728     if (r < 0)
2729       return r;
2730
2731     if (s->cct->_conf->subsys.should_gather<ceph_subsys_rgw, 20>()) {
2732       ldpp_dout(this, 20) << "read part header -- part.name="
2733                         << part.name << dendl;
2734
2735       for (const auto& pair : part.fields) {
2736         ldpp_dout(this, 20) << "field.name=" << pair.first << dendl;
2737         ldpp_dout(this, 20) << "field.val=" << pair.second.val << dendl;
2738         ldpp_dout(this, 20) << "field.params:" << dendl;
2739
2740         for (const auto& param_pair : pair.second.params) {
2741           ldpp_dout(this, 20) << " " << param_pair.first
2742                             << " -> " << param_pair.second << dendl;
2743         }
2744       }
2745     }
2746
2747     if (done) { /* unexpected here */
2748       err_msg = "Malformed request";
2749       return -EINVAL;
2750     }
2751
2752     if (stringcasecmp(part.name, "file") == 0) { /* beginning of data transfer */
2753       struct post_part_field& field = part.fields["Content-Disposition"];
2754       map<string, string>::iterator iter = field.params.find("filename");
2755       if (iter != field.params.end()) {
2756         filename = iter->second;
2757       }
2758       parts[part.name] = part;
2759       break;
2760     }
2761
2762     bool boundary;
2763     uint64_t chunk_size = s->cct->_conf->rgw_max_chunk_size;
2764     r = read_data(part.data, chunk_size, boundary, done);
2765     if (r < 0 || !boundary) {
2766       err_msg = "Couldn't find boundary";
2767       return -EINVAL;
2768     }
2769     parts[part.name] = part;
2770     string part_str(part.data.c_str(), part.data.length());
2771     env.add_var(part.name, part_str);
2772   } while (!done);
2773
2774   string object_str;
2775   if (!part_str(parts, "key", &object_str)) {
2776     err_msg = "Key not specified";
2777     return -EINVAL;
2778   }
2779
2780   s->object = rgw_obj_key(object_str);
2781
2782   rebuild_key(s->object.name);
2783
2784   if (s->object.empty()) {
2785     err_msg = "Empty object name";
2786     return -EINVAL;
2787   }
2788
2789   env.add_var("key", s->object.name);
2790
2791   part_str(parts, "Content-Type", &content_type);
2792
2793   /* AWS permits POST without Content-Type: http://tracker.ceph.com/issues/20201 */
2794   if (! content_type.empty()) {
2795     env.add_var("Content-Type", content_type);
2796   }
2797
2798   std::string storage_class;
2799   part_str(parts, "x-amz-storage-class", &storage_class);
2800
2801   if (! storage_class.empty()) {
2802     s->dest_placement.storage_class = storage_class;
2803     if (!store->svc()->zone->get_zone_params().valid_placement(s->dest_placement)) {
2804       ldpp_dout(this, 0) << "NOTICE: invalid dest placement: " << s->dest_placement.to_str() << dendl;
2805       err_msg = "The storage class you specified is not valid";
2806       return -EINVAL;
2807     }
2808   }
2809
2810   map<string, struct post_form_part, ltstr_nocase>::iterator piter =
2811     parts.upper_bound(RGW_AMZ_META_PREFIX);
2812   for (; piter != parts.end(); ++piter) {
2813     string n = piter->first;
2814     if (strncasecmp(n.c_str(), RGW_AMZ_META_PREFIX,
2815                     sizeof(RGW_AMZ_META_PREFIX) - 1) != 0)
2816       break;
2817
2818     string attr_name = RGW_ATTR_PREFIX;
2819     attr_name.append(n);
2820
2821     /* need to null terminate it */
2822     bufferlist& data = piter->second.data;
2823     string str = string(data.c_str(), data.length());
2824
2825     bufferlist attr_bl;
2826     attr_bl.append(str.c_str(), str.size() + 1);
2827
2828     attrs[attr_name] = attr_bl;
2829   }
2830   // TODO: refactor this and the above loop to share code
2831   piter = parts.find(RGW_AMZ_WEBSITE_REDIRECT_LOCATION);
2832   if (piter != parts.end()) {
2833     string n = piter->first;
2834     string attr_name = RGW_ATTR_PREFIX;
2835     attr_name.append(n);
2836     /* need to null terminate it */
2837     bufferlist& data = piter->second.data;
2838     string str = string(data.c_str(), data.length());
2839
2840     bufferlist attr_bl;
2841     attr_bl.append(str.c_str(), str.size() + 1);
2842
2843     attrs[attr_name] = attr_bl;
2844   }
2845
2846   int r = get_policy();
2847   if (r < 0)
2848     return r;
2849
2850   r = get_tags();
2851   if (r < 0)
2852     return r;
2853
2854
2855   min_len = post_policy.min_length;
2856   max_len = post_policy.max_length;
2857
2858
2859
2860   return 0;
2861 }
2862
2863 int RGWPostObj_ObjStore_S3::get_tags()
2864 {
2865   string tags_str;
2866   if (part_str(parts, "tagging", &tags_str)) {
2867     RGWXMLParser parser;
2868     if (!parser.init()){
2869       ldpp_dout(this, 0) << "Couldn't init RGWObjTags XML parser" << dendl;
2870       err_msg = "Server couldn't process the request";
2871       return -EINVAL; // TODO: This class of errors in rgw code should be a 5XX error
2872     }
2873     if (!parser.parse(tags_str.c_str(), tags_str.size(), 1)) {
2874       ldpp_dout(this,0 ) << "Invalid Tagging XML" << dendl;
2875       err_msg = "Invalid Tagging XML";
2876       return -EINVAL;
2877     }
2878
2879     RGWObjTagging_S3 tagging;
2880
2881     try {
2882       RGWXMLDecoder::decode_xml("Tagging", tagging, &parser);
2883     } catch (RGWXMLDecoder::err& err) {
2884       ldpp_dout(this, 5) << "Malformed tagging request: " << err << dendl;
2885       return -EINVAL;
2886     }
2887
2888     RGWObjTags obj_tags;
2889     int r = tagging.rebuild(obj_tags);
2890     if (r < 0)
2891       return r;
2892
2893     bufferlist tags_bl;
2894     obj_tags.encode(tags_bl);
2895     ldpp_dout(this, 20) << "Read " << obj_tags.count() << "tags" << dendl;
2896     attrs[RGW_ATTR_TAGS] = tags_bl;
2897   }
2898
2899
2900   return 0;
2901 }
2902
2903 int RGWPostObj_ObjStore_S3::get_policy()
2904 {
2905   if (part_bl(parts, "policy", &s->auth.s3_postobj_creds.encoded_policy)) {
2906     bool aws4_auth = false;
2907
2908     /* x-amz-algorithm handling */
2909     using rgw::auth::s3::AWS4_HMAC_SHA256_STR;
2910     if ((part_str(parts, "x-amz-algorithm", &s->auth.s3_postobj_creds.x_amz_algorithm)) &&
2911         (s->auth.s3_postobj_creds.x_amz_algorithm == AWS4_HMAC_SHA256_STR)) {
2912       ldpp_dout(this, 0) << "Signature verification algorithm AWS v4 (AWS4-HMAC-SHA256)" << dendl;
2913       aws4_auth = true;
2914     } else {
2915       ldpp_dout(this, 0) << "Signature verification algorithm AWS v2" << dendl;
2916     }
2917
2918     // check that the signature matches the encoded policy
2919     if (aws4_auth) {
2920       /* AWS4 */
2921
2922       /* x-amz-credential handling */
2923       if (!part_str(parts, "x-amz-credential",
2924                     &s->auth.s3_postobj_creds.x_amz_credential)) {
2925         ldpp_dout(this, 0) << "No S3 aws4 credential found!" << dendl;
2926         err_msg = "Missing aws4 credential";
2927         return -EINVAL;
2928       }
2929
2930       /* x-amz-signature handling */
2931       if (!part_str(parts, "x-amz-signature",
2932                     &s->auth.s3_postobj_creds.signature)) {
2933         ldpp_dout(this, 0) << "No aws4 signature found!" << dendl;
2934         err_msg = "Missing aws4 signature";
2935         return -EINVAL;
2936       }
2937
2938       /* x-amz-date handling */
2939       std::string received_date_str;
2940       if (!part_str(parts, "x-amz-date", &received_date_str)) {
2941         ldpp_dout(this, 0) << "No aws4 date found!" << dendl;
2942         err_msg = "Missing aws4 date";
2943         return -EINVAL;
2944       }
2945     } else {
2946       /* AWS2 */
2947
2948       // check that the signature matches the encoded policy
2949       if (!part_str(parts, "AWSAccessKeyId",
2950                     &s->auth.s3_postobj_creds.access_key)) {
2951         ldpp_dout(this, 0) << "No S3 aws2 access key found!" << dendl;
2952         err_msg = "Missing aws2 access key";
2953         return -EINVAL;
2954       }
2955
2956       if (!part_str(parts, "signature", &s->auth.s3_postobj_creds.signature)) {
2957         ldpp_dout(this, 0) << "No aws2 signature found!" << dendl;
2958         err_msg = "Missing aws2 signature";
2959         return -EINVAL;
2960       }
2961     }
2962
2963     if (part_str(parts, "x-amz-security-token", &s->auth.s3_postobj_creds.x_amz_security_token)) {
2964       if (s->auth.s3_postobj_creds.x_amz_security_token.size() == 0) {
2965         err_msg = "Invalid token";
2966         return -EINVAL;
2967       }
2968     }
2969
2970     /* FIXME: this is a makeshift solution. The browser upload authentication will be
2971      * handled by an instance of rgw::auth::Completer spawned in Handler's authorize()
2972      * method. */
2973     const int ret = rgw::auth::Strategy::apply(this, auth_registry_ptr->get_s3_post(), s);
2974     if (ret != 0) {
2975       return -EACCES;
2976     } else {
2977       /* Populate the owner info. */
2978       s->owner.set_id(s->user->get_id());
2979       s->owner.set_name(s->user->get_display_name());
2980       ldpp_dout(this, 20) << "Successful Signature Verification!" << dendl;
2981     }
2982
2983     ceph::bufferlist decoded_policy;
2984     try {
2985       decoded_policy.decode_base64(s->auth.s3_postobj_creds.encoded_policy);
2986     } catch (buffer::error& err) {
2987       ldpp_dout(this, 0) << "failed to decode_base64 policy" << dendl;
2988       err_msg = "Could not decode policy";
2989       return -EINVAL;
2990     }
2991
2992     decoded_policy.append('\0'); // NULL terminate
2993     ldpp_dout(this, 20) << "POST policy: " << decoded_policy.c_str() << dendl;
2994
2995
2996     int r = post_policy.from_json(decoded_policy, err_msg);
2997     if (r < 0) {
2998       if (err_msg.empty()) {
2999         err_msg = "Failed to parse policy";
3000       }
3001       ldpp_dout(this, 0) << "failed to parse policy" << dendl;
3002       return -EINVAL;
3003     }
3004
3005     if (aws4_auth) {
3006       /* AWS4 */
3007       post_policy.set_var_checked("x-amz-signature");
3008     } else {
3009       /* AWS2 */
3010       post_policy.set_var_checked("AWSAccessKeyId");
3011       post_policy.set_var_checked("signature");
3012     }
3013     post_policy.set_var_checked("policy");
3014
3015     r = post_policy.check(&env, err_msg);
3016     if (r < 0) {
3017       if (err_msg.empty()) {
3018         err_msg = "Policy check failed";
3019       }
3020       ldpp_dout(this, 0) << "policy check failed" << dendl;
3021       return r;
3022     }
3023
3024   } else {
3025     ldpp_dout(this, 0) << "No attached policy found!" << dendl;
3026   }
3027
3028   string canned_acl;
3029   part_str(parts, "acl", &canned_acl);
3030
3031   RGWAccessControlPolicy_S3 s3policy(s->cct);
3032   ldpp_dout(this, 20) << "canned_acl=" << canned_acl << dendl;
3033   if (s3policy.create_canned(s->owner, s->bucket_owner, canned_acl) < 0) {
3034     err_msg = "Bad canned ACLs";
3035     return -EINVAL;
3036   }
3037
3038   policy = s3policy;
3039
3040   return 0;
3041 }
3042
3043 int RGWPostObj_ObjStore_S3::complete_get_params()
3044 {
3045   bool done;
3046   do {
3047     struct post_form_part part;
3048     int r = read_form_part_header(&part, done);
3049     if (r < 0) {
3050       return r;
3051     }
3052
3053     ceph::bufferlist part_data;
3054     bool boundary;
3055     uint64_t chunk_size = s->cct->_conf->rgw_max_chunk_size;
3056     r = read_data(part.data, chunk_size, boundary, done);
3057     if (r < 0 || !boundary) {
3058       return -EINVAL;
3059     }
3060
3061     /* Just reading the data but not storing any results of that. */
3062   } while (!done);
3063
3064   return 0;
3065 }
3066
3067 int RGWPostObj_ObjStore_S3::get_data(ceph::bufferlist& bl, bool& again)
3068 {
3069   bool boundary;
3070   bool done;
3071
3072   const uint64_t chunk_size = s->cct->_conf->rgw_max_chunk_size;
3073   int r = read_data(bl, chunk_size, boundary, done);
3074   if (r < 0) {
3075     return r;
3076   }
3077
3078   if (boundary) {
3079     if (!done) {
3080       /* Reached end of data, let's drain the rest of the params */
3081       r = complete_get_params();
3082       if (r < 0) {
3083        return r;
3084       }
3085     }
3086   }
3087
3088   again = !boundary;
3089   return bl.length();
3090 }
3091
3092 void RGWPostObj_ObjStore_S3::send_response()
3093 {
3094   if (op_ret == 0 && parts.count("success_action_redirect")) {
3095     string redirect;
3096
3097     part_str(parts, "success_action_redirect", &redirect);
3098
3099     string tenant;
3100     string bucket;
3101     string key;
3102     string etag_str = "\"";
3103
3104     etag_str.append(etag);
3105     etag_str.append("\"");
3106
3107     string etag_url;
3108
3109     url_encode(s->bucket_tenant, tenant); /* surely overkill, but cheap */
3110     url_encode(s->bucket_name, bucket);
3111     url_encode(s->object.name, key);
3112     url_encode(etag_str, etag_url);
3113
3114     if (!s->bucket_tenant.empty()) {
3115       /*
3116        * What we really would like is to quaily the bucket name, so
3117        * that the client could simply copy it and paste into next request.
3118        * Unfortunately, in S3 we cannot know if the client will decide
3119        * to come through DNS, with "bucket.tenant" sytanx, or through
3120        * URL with "tenant\bucket" syntax. Therefore, we provide the
3121        * tenant separately.
3122        */
3123       redirect.append("?tenant=");
3124       redirect.append(tenant);
3125       redirect.append("&bucket=");
3126       redirect.append(bucket);
3127     } else {
3128       redirect.append("?bucket=");
3129       redirect.append(bucket);
3130     }
3131     redirect.append("&key=");
3132     redirect.append(key);
3133     redirect.append("&etag=");
3134     redirect.append(etag_url);
3135
3136     int r = check_utf8(redirect.c_str(), redirect.size());
3137     if (r < 0) {
3138       op_ret = r;
3139       goto done;
3140     }
3141     dump_redirect(s, redirect);
3142     op_ret = STATUS_REDIRECT;
3143   } else if (op_ret == 0 && parts.count("success_action_status")) {
3144     string status_string;
3145     uint32_t status_int;
3146
3147     part_str(parts, "success_action_status", &status_string);
3148
3149     int r = stringtoul(status_string, &status_int);
3150     if (r < 0) {
3151       op_ret = r;
3152       goto done;
3153     }
3154
3155     switch (status_int) {
3156       case 200:
3157         break;
3158       case 201:
3159         op_ret = STATUS_CREATED;
3160         break;
3161       default:
3162         op_ret = STATUS_NO_CONTENT;
3163         break;
3164     }
3165   } else if (! op_ret) {
3166     op_ret = STATUS_NO_CONTENT;
3167   }
3168
3169 done:
3170   if (op_ret == STATUS_CREATED) {
3171     for (auto &it : crypt_http_responses)
3172       dump_header(s, it.first, it.second);
3173     s->formatter->open_object_section("PostResponse");
3174     std::string base_uri = compute_domain_uri(s);
3175     if (!s->bucket_tenant.empty()){
3176       s->formatter->dump_format("Location", "%s/%s:%s/%s",
3177                                 base_uri.c_str(),
3178                                 url_encode(s->bucket_tenant).c_str(),
3179                                 url_encode(s->bucket_name).c_str(),
3180                                 url_encode(s->object.name).c_str());
3181       s->formatter->dump_string("Tenant", s->bucket_tenant);
3182     } else {
3183       s->formatter->dump_format("Location", "%s/%s/%s",
3184                                 base_uri.c_str(),
3185                                 url_encode(s->bucket_name).c_str(),
3186                                 url_encode(s->object.name).c_str());
3187     }
3188     s->formatter->dump_string("Bucket", s->bucket_name);
3189     s->formatter->dump_string("Key", s->object.name);
3190     s->formatter->dump_string("ETag", etag);
3191     s->formatter->close_section();
3192   }
3193   s->err.message = err_msg;
3194   set_req_state_err(s, op_ret);
3195   dump_errno(s);
3196   if (op_ret >= 0) {
3197     dump_content_length(s, s->formatter->get_len());
3198   }
3199   end_header(s, this);
3200   if (op_ret != STATUS_CREATED)
3201     return;
3202
3203   rgw_flush_formatter_and_reset(s, s->formatter);
3204 }
3205
3206 int RGWPostObj_ObjStore_S3::get_encrypt_filter(
3207     std::unique_ptr<rgw::putobj::DataProcessor> *filter,
3208     rgw::putobj::DataProcessor *cb)
3209 {
3210   std::unique_ptr<BlockCrypt> block_crypt;
3211   int res = rgw_s3_prepare_encrypt(s, attrs, &parts, &block_crypt,
3212                                    crypt_http_responses);
3213   if (res == 0 && block_crypt != nullptr) {
3214     filter->reset(new RGWPutObj_BlockEncrypt(s->cct, cb, std::move(block_crypt)));
3215   }
3216   return res;
3217 }
3218
3219 int RGWDeleteObj_ObjStore_S3::get_params()
3220 {
3221   const char *if_unmod = s->info.env->get("HTTP_X_AMZ_DELETE_IF_UNMODIFIED_SINCE");
3222
3223   if (s->system_request) {
3224     s->info.args.get_bool(RGW_SYS_PARAM_PREFIX "no-precondition-error", &no_precondition_error, false);
3225   }
3226
3227   if (if_unmod) {
3228     std::string if_unmod_decoded = url_decode(if_unmod);
3229     uint64_t epoch;
3230     uint64_t nsec;
3231     if (utime_t::parse_date(if_unmod_decoded, &epoch, &nsec) < 0) {
3232       ldpp_dout(this, 10) << "failed to parse time: " << if_unmod_decoded << dendl;
3233       return -EINVAL;
3234     }
3235     unmod_since = utime_t(epoch, nsec).to_real_time();
3236   }
3237
3238   const char *bypass_gov_header = s->info.env->get("HTTP_X_AMZ_BYPASS_GOVERNANCE_RETENTION");
3239   if (bypass_gov_header) {
3240     std::string bypass_gov_decoded = url_decode(bypass_gov_header);
3241     bypass_governance_mode = boost::algorithm::iequals(bypass_gov_decoded, "true");
3242   }
3243
3244   return 0;
3245 }
3246
3247 void RGWDeleteObj_ObjStore_S3::send_response()
3248 {
3249   int r = op_ret;
3250   if (r == -ENOENT)
3251     r = 0;
3252   if (!r)
3253     r = STATUS_NO_CONTENT;
3254
3255   set_req_state_err(s, r);
3256   dump_errno(s);
3257   dump_header_if_nonempty(s, "x-amz-version-id", version_id);
3258   if (delete_marker) {
3259     dump_header(s, "x-amz-delete-marker", "true");
3260   }
3261   end_header(s, this);
3262 }
3263
3264 int RGWCopyObj_ObjStore_S3::init_dest_policy()
3265 {
3266   RGWAccessControlPolicy_S3 s3policy(s->cct);
3267
3268   /* build a policy for the target object */
3269   int r = create_s3_policy(s, store, s3policy, s->owner);
3270   if (r < 0)
3271     return r;
3272
3273   dest_policy = s3policy;
3274
3275   return 0;
3276 }
3277
3278 int RGWCopyObj_ObjStore_S3::get_params()
3279 {
3280   if_mod = s->info.env->get("HTTP_X_AMZ_COPY_IF_MODIFIED_SINCE");
3281   if_unmod = s->info.env->get("HTTP_X_AMZ_COPY_IF_UNMODIFIED_SINCE");
3282   if_match = s->info.env->get("HTTP_X_AMZ_COPY_IF_MATCH");
3283   if_nomatch = s->info.env->get("HTTP_X_AMZ_COPY_IF_NONE_MATCH");
3284
3285   src_tenant_name = s->src_tenant_name;
3286   src_bucket_name = s->src_bucket_name;
3287   src_object = s->src_object;
3288   dest_tenant_name = s->bucket.tenant;
3289   dest_bucket_name = s->bucket.name;
3290   dest_object = s->object.name;
3291
3292   if (s->system_request) {
3293     source_zone = s->info.args.get(RGW_SYS_PARAM_PREFIX "source-zone");
3294     s->info.args.get_bool(RGW_SYS_PARAM_PREFIX "copy-if-newer", &copy_if_newer, false);
3295   }
3296
3297   copy_source = s->info.env->get("HTTP_X_AMZ_COPY_SOURCE");
3298   auto tmp_md_d = s->info.env->get("HTTP_X_AMZ_METADATA_DIRECTIVE");
3299   if (tmp_md_d) {
3300     if (strcasecmp(tmp_md_d, "COPY") == 0) {
3301       attrs_mod = RGWRados::ATTRSMOD_NONE;
3302     } else if (strcasecmp(tmp_md_d, "REPLACE") == 0) {
3303       attrs_mod = RGWRados::ATTRSMOD_REPLACE;
3304     } else if (!source_zone.empty()) {
3305       attrs_mod = RGWRados::ATTRSMOD_NONE; // default for intra-zone_group copy
3306     } else {
3307       s->err.message = "Unknown metadata directive.";
3308       ldpp_dout(this, 0) << s->err.message << dendl;
3309       return -EINVAL;
3310     }
3311     md_directive = tmp_md_d;
3312   }
3313
3314   if (source_zone.empty() &&
3315       (dest_tenant_name.compare(src_tenant_name) == 0) &&
3316       (dest_bucket_name.compare(src_bucket_name) == 0) &&
3317       (dest_object.compare(src_object.name) == 0) &&
3318       src_object.instance.empty() &&
3319       (attrs_mod != RGWRados::ATTRSMOD_REPLACE)) {
3320     need_to_check_storage_class = true;
3321   }
3322
3323   return 0;
3324 }
3325
3326 int RGWCopyObj_ObjStore_S3::check_storage_class(const rgw_placement_rule& src_placement)
3327 {
3328   if (src_placement == s->dest_placement) {
3329     /* can only copy object into itself if replacing attrs */
3330     s->err.message = "This copy request is illegal because it is trying to copy "
3331       "an object to itself without changing the object's metadata, "
3332       "storage class, website redirect location or encryption attributes.";
3333     ldpp_dout(this, 0) << s->err.message << dendl;
3334     return -ERR_INVALID_REQUEST;
3335   }
3336   return 0;
3337 }
3338
3339 void RGWCopyObj_ObjStore_S3::send_partial_response(off_t ofs)
3340 {
3341   if (! sent_header) {
3342     if (op_ret)
3343     set_req_state_err(s, op_ret);
3344     dump_errno(s);
3345
3346     // Explicitly use chunked transfer encoding so that we can stream the result
3347     // to the user without having to wait for the full length of it.
3348     end_header(s, this, "application/xml", CHUNKED_TRANSFER_ENCODING);
3349     dump_start(s);
3350     if (op_ret == 0) {
3351       s->formatter->open_object_section_in_ns("CopyObjectResult", XMLNS_AWS_S3);
3352     }
3353     sent_header = true;
3354   } else {
3355     /* Send progress field. Note that this diverge from the original S3
3356      * spec. We do this in order to keep connection alive.
3357      */
3358     s->formatter->dump_int("Progress", (uint64_t)ofs);
3359   }
3360   rgw_flush_formatter(s, s->formatter);
3361 }
3362
3363 void RGWCopyObj_ObjStore_S3::send_response()
3364 {
3365   if (!sent_header)
3366     send_partial_response(0);
3367
3368   if (op_ret == 0) {
3369     dump_time(s, "LastModified", &mtime);
3370     if (!etag.empty()) {
3371       s->formatter->dump_string("ETag", std::move(etag));
3372     }
3373     s->formatter->close_section();
3374     rgw_flush_formatter_and_reset(s, s->formatter);
3375   }
3376 }
3377
3378 void RGWGetACLs_ObjStore_S3::send_response()
3379 {
3380   if (op_ret)
3381     set_req_state_err(s, op_ret);
3382   dump_errno(s);
3383   end_header(s, this, "application/xml");
3384   dump_start(s);
3385   rgw_flush_formatter(s, s->formatter);
3386   dump_body(s, acls);
3387 }
3388
3389 int RGWPutACLs_ObjStore_S3::get_params()
3390 {
3391   int ret =  RGWPutACLs_ObjStore::get_params();
3392   if (ret >= 0) {
3393     const int ret_auth = do_aws4_auth_completion();
3394     if (ret_auth < 0) {
3395       return ret_auth;
3396     }
3397   } else {
3398     /* a request body is not required an S3 PutACLs request--n.b.,
3399      * s->length is non-null iff a content length was parsed (the
3400      * ACP or canned ACL could be in any of 3 headers, don't worry
3401      * about that here) */
3402     if ((ret == -ERR_LENGTH_REQUIRED) &&
3403         !!(s->length)) {
3404       return 0;
3405     }
3406   }
3407   return ret;
3408 }
3409
3410 int RGWPutACLs_ObjStore_S3::get_policy_from_state(rgw::sal::RGWRadosStore *store,
3411                                                   struct req_state *s,
3412                                                   stringstream& ss)
3413 {
3414   RGWAccessControlPolicy_S3 s3policy(s->cct);
3415
3416   // bucket-* canned acls do not apply to bucket
3417   if (s->object.empty()) {
3418     if (s->canned_acl.find("bucket") != string::npos)
3419       s->canned_acl.clear();
3420   }
3421
3422   int r = create_s3_policy(s, store, s3policy, owner);
3423   if (r < 0)
3424     return r;
3425
3426   s3policy.to_xml(ss);
3427
3428   return 0;
3429 }
3430
3431 void RGWPutACLs_ObjStore_S3::send_response()
3432 {
3433   if (op_ret)
3434     set_req_state_err(s, op_ret);
3435   dump_errno(s);
3436   end_header(s, this, "application/xml");
3437   dump_start(s);
3438 }
3439
3440 void RGWGetLC_ObjStore_S3::execute()
3441 {
3442   config.set_ctx(s->cct);
3443
3444   map<string, bufferlist>::iterator aiter = s->bucket_attrs.find(RGW_ATTR_LC);
3445   if (aiter == s->bucket_attrs.end()) {
3446     op_ret = -ENOENT;
3447     return;
3448   }
3449
3450   bufferlist::const_iterator iter{&aiter->second};
3451   try {
3452       config.decode(iter);
3453     } catch (const buffer::error& e) {
3454       ldpp_dout(this, 0) << __func__ <<  "decode life cycle config failed" << dendl;
3455       op_ret = -EIO;
3456       return;
3457     }
3458 }
3459
3460 void RGWGetLC_ObjStore_S3::send_response()
3461 {
3462   if (op_ret) {
3463     if (op_ret == -ENOENT) {
3464       set_req_state_err(s, ERR_NO_SUCH_LC);
3465     } else {
3466       set_req_state_err(s, op_ret);
3467     }
3468   }
3469   dump_errno(s);
3470   end_header(s, this, "application/xml");
3471   dump_start(s);
3472
3473   if (op_ret < 0)
3474     return;
3475
3476   encode_xml("LifecycleConfiguration", XMLNS_AWS_S3, config, s->formatter);
3477   rgw_flush_formatter_and_reset(s, s->formatter);
3478 }
3479
3480 void RGWPutLC_ObjStore_S3::send_response()
3481 {
3482   if (op_ret)
3483     set_req_state_err(s, op_ret);
3484   dump_errno(s);
3485   end_header(s, this, "application/xml");
3486   dump_start(s);
3487 }
3488
3489 void RGWDeleteLC_ObjStore_S3::send_response()
3490 {
3491   if (op_ret == 0)
3492       op_ret = STATUS_NO_CONTENT;
3493   if (op_ret) {
3494     set_req_state_err(s, op_ret);
3495   }
3496   dump_errno(s);
3497   end_header(s, this, "application/xml");
3498   dump_start(s);
3499 }
3500
3501 void RGWGetCORS_ObjStore_S3::send_response()
3502 {
3503   if (op_ret) {
3504     if (op_ret == -ENOENT)
3505       set_req_state_err(s, ERR_NO_SUCH_CORS_CONFIGURATION);
3506     else
3507       set_req_state_err(s, op_ret);
3508   }
3509   dump_errno(s);
3510   end_header(s, NULL, "application/xml");
3511   dump_start(s);
3512   if (! op_ret) {
3513     string cors;
3514     RGWCORSConfiguration_S3 *s3cors =
3515       static_cast<RGWCORSConfiguration_S3 *>(&bucket_cors);
3516     stringstream ss;
3517
3518     s3cors->to_xml(ss);
3519     cors = ss.str();
3520     dump_body(s, cors);
3521   }
3522 }
3523
3524 int RGWPutCORS_ObjStore_S3::get_params()
3525 {
3526   RGWCORSXMLParser_S3 parser(s->cct);
3527   RGWCORSConfiguration_S3 *cors_config;
3528
3529   const auto max_size = s->cct->_conf->rgw_max_put_param_size;
3530
3531   int r = 0;
3532   bufferlist data;
3533   std::tie(r, data) = rgw_rest_read_all_input(s, max_size, false);
3534   if (r < 0) {
3535     return r;
3536   }
3537
3538   r = do_aws4_auth_completion();
3539   if (r < 0) {
3540     return r;
3541   }
3542
3543   if (!parser.init()) {
3544     return -EINVAL;
3545   }
3546
3547   char* buf = data.c_str();
3548   if (!buf || !parser.parse(buf, data.length(), 1)) {
3549     return -ERR_MALFORMED_XML;
3550   }
3551   cors_config =
3552     static_cast<RGWCORSConfiguration_S3 *>(parser.find_first(
3553                                              "CORSConfiguration"));
3554   if (!cors_config) {
3555     return -ERR_MALFORMED_XML;
3556   }
3557
3558 #define CORS_RULES_MAX_NUM      100
3559   int max_num = s->cct->_conf->rgw_cors_rules_max_num;
3560   if (max_num < 0) {
3561     max_num = CORS_RULES_MAX_NUM;
3562   }
3563   int cors_rules_num = cors_config->get_rules().size();
3564   if (cors_rules_num > max_num) {
3565     ldpp_dout(this, 4) << "An cors config can have up to "
3566                      << max_num
3567                      << " rules, request cors rules num: "
3568                      << cors_rules_num << dendl;
3569     op_ret = -ERR_INVALID_CORS_RULES_ERROR;
3570     s->err.message = "The number of CORS rules should not exceed allowed limit of "
3571                      + std::to_string(max_num) + " rules.";
3572     return -ERR_INVALID_REQUEST;
3573   }
3574
3575   // forward bucket cors requests to meta master zone
3576   if (!store->svc()->zone->is_meta_master()) {
3577     /* only need to keep this data around if we're not meta master */
3578     in_data.append(data);
3579   }
3580
3581   if (s->cct->_conf->subsys.should_gather<ceph_subsys_rgw, 15>()) {
3582     ldpp_dout(this, 15) << "CORSConfiguration";
3583     cors_config->to_xml(*_dout);
3584     *_dout << dendl;
3585   }
3586
3587   cors_config->encode(cors_bl);
3588
3589   return 0;
3590 }
3591
3592 void RGWPutCORS_ObjStore_S3::send_response()
3593 {
3594   if (op_ret)
3595     set_req_state_err(s, op_ret);
3596   dump_errno(s);
3597   end_header(s, NULL, "application/xml");
3598   dump_start(s);
3599 }
3600
3601 void RGWDeleteCORS_ObjStore_S3::send_response()
3602 {
3603   int r = op_ret;
3604   if (!r || r == -ENOENT)
3605     r = STATUS_NO_CONTENT;
3606
3607   set_req_state_err(s, r);
3608   dump_errno(s);
3609   end_header(s, NULL);
3610 }
3611
3612 void RGWOptionsCORS_ObjStore_S3::send_response()
3613 {
3614   string hdrs, exp_hdrs;
3615   uint32_t max_age = CORS_MAX_AGE_INVALID;
3616   /*EACCES means, there is no CORS registered yet for the bucket
3617    *ENOENT means, there is no match of the Origin in the list of CORSRule
3618    */
3619   if (op_ret == -ENOENT)
3620     op_ret = -EACCES;
3621   if (op_ret < 0) {
3622     set_req_state_err(s, op_ret);
3623     dump_errno(s);
3624     end_header(s, NULL);
3625     return;
3626   }
3627   get_response_params(hdrs, exp_hdrs, &max_age);
3628
3629   dump_errno(s);
3630   dump_access_control(s, origin, req_meth, hdrs.c_str(), exp_hdrs.c_str(),
3631                       max_age);
3632   end_header(s, NULL);
3633 }
3634
3635 void RGWGetRequestPayment_ObjStore_S3::send_response()
3636 {
3637   dump_errno(s);
3638   end_header(s, this, "application/xml");
3639   dump_start(s);
3640
3641   s->formatter->open_object_section_in_ns("RequestPaymentConfiguration", XMLNS_AWS_S3);
3642   const char *payer = requester_pays ? "Requester" :  "BucketOwner";
3643   s->formatter->dump_string("Payer", payer);
3644   s->formatter->close_section();
3645   rgw_flush_formatter_and_reset(s, s->formatter);
3646 }
3647
3648 class RGWSetRequestPaymentParser : public RGWXMLParser
3649 {
3650   XMLObj *alloc_obj(const char *el) override {
3651     return new XMLObj;
3652   }
3653
3654 public:
3655   RGWSetRequestPaymentParser() {}
3656   ~RGWSetRequestPaymentParser() override {}
3657
3658   int get_request_payment_payer(bool *requester_pays) {
3659     XMLObj *config = find_first("RequestPaymentConfiguration");
3660     if (!config)
3661       return -EINVAL;
3662
3663     *requester_pays = false;
3664
3665     XMLObj *field = config->find_first("Payer");
3666     if (!field)
3667       return 0;
3668
3669     auto& s = field->get_data();
3670
3671     if (stringcasecmp(s, "Requester") == 0) {
3672       *requester_pays = true;
3673     } else if (stringcasecmp(s, "BucketOwner") != 0) {
3674       return -EINVAL;
3675     }
3676
3677     return 0;
3678   }
3679 };
3680
3681 int RGWSetRequestPayment_ObjStore_S3::get_params()
3682 {
3683   const auto max_size = s->cct->_conf->rgw_max_put_param_size;
3684
3685   int r = 0;
3686   std::tie(r, in_data) = rgw_rest_read_all_input(s, max_size, false);
3687
3688   if (r < 0) {
3689     return r;
3690   }
3691
3692
3693   RGWSetRequestPaymentParser parser;
3694
3695   if (!parser.init()) {
3696     ldpp_dout(this, 0) << "ERROR: failed to initialize parser" << dendl;
3697     return -EIO;
3698   }
3699
3700   char* buf = in_data.c_str();
3701   if (!parser.parse(buf, in_data.length(), 1)) {
3702     ldpp_dout(this, 10) << "failed to parse data: " << buf << dendl;
3703     return -EINVAL;
3704   }
3705
3706   return parser.get_request_payment_payer(&requester_pays);
3707 }
3708
3709 void RGWSetRequestPayment_ObjStore_S3::send_response()
3710 {
3711   if (op_ret)
3712     set_req_state_err(s, op_ret);
3713   dump_errno(s);
3714   end_header(s);
3715 }
3716
3717 int RGWInitMultipart_ObjStore_S3::get_params()
3718 {
3719   RGWAccessControlPolicy_S3 s3policy(s->cct);
3720   op_ret = create_s3_policy(s, store, s3policy, s->owner);
3721   if (op_ret < 0)
3722     return op_ret;
3723
3724   policy = s3policy;
3725
3726   return 0;
3727 }
3728
3729 void RGWInitMultipart_ObjStore_S3::send_response()
3730 {
3731   if (op_ret)
3732     set_req_state_err(s, op_ret);
3733   dump_errno(s);
3734   for (auto &it : crypt_http_responses)
3735      dump_header(s, it.first, it.second);
3736   ceph::real_time abort_date;
3737   string rule_id;
3738   bool exist_multipart_abort = get_s3_multipart_abort_header(s, mtime, abort_date, rule_id);
3739   if (exist_multipart_abort) {
3740     dump_time_header(s, "x-amz-abort-date", abort_date);
3741     dump_header_if_nonempty(s, "x-amz-abort-rule-id", rule_id);
3742   }
3743   end_header(s, this, "application/xml");
3744   if (op_ret == 0) {
3745     dump_start(s);
3746     s->formatter->open_object_section_in_ns("InitiateMultipartUploadResult", XMLNS_AWS_S3);
3747     if (!s->bucket_tenant.empty())
3748       s->formatter->dump_string("Tenant", s->bucket_tenant);
3749     s->formatter->dump_string("Bucket", s->bucket_name);
3750     s->formatter->dump_string("Key", s->object.name);
3751     s->formatter->dump_string("UploadId", upload_id);
3752     s->formatter->close_section();
3753     rgw_flush_formatter_and_reset(s, s->formatter);
3754   }
3755 }
3756
3757 int RGWInitMultipart_ObjStore_S3::prepare_encryption(map<string, bufferlist>& attrs)
3758 {
3759   int res = 0;
3760   res = rgw_s3_prepare_encrypt(s, attrs, nullptr, nullptr, crypt_http_responses);
3761   return res;
3762 }
3763
3764 int RGWCompleteMultipart_ObjStore_S3::get_params()
3765 {
3766   int ret = RGWCompleteMultipart_ObjStore::get_params();
3767   if (ret < 0) {
3768     return ret;
3769   }
3770
3771   map_qs_metadata(s);
3772
3773   return do_aws4_auth_completion();
3774 }
3775
3776 void RGWCompleteMultipart_ObjStore_S3::send_response()
3777 {
3778   if (op_ret)
3779     set_req_state_err(s, op_ret);
3780   dump_errno(s);
3781   dump_header_if_nonempty(s, "x-amz-version-id", version_id);
3782   end_header(s, this, "application/xml");
3783   if (op_ret == 0) {
3784     dump_start(s);
3785     s->formatter->open_object_section_in_ns("CompleteMultipartUploadResult", XMLNS_AWS_S3);
3786     std::string base_uri = compute_domain_uri(s);
3787     if (!s->bucket_tenant.empty()) {
3788       s->formatter->dump_format("Location", "%s/%s:%s/%s",
3789           base_uri.c_str(),
3790           s->bucket_tenant.c_str(),
3791           s->bucket_name.c_str(),
3792           s->object.name.c_str()
3793           );
3794       s->formatter->dump_string("Tenant", s->bucket_tenant);
3795     } else {
3796       s->formatter->dump_format("Location", "%s/%s/%s",
3797           base_uri.c_str(),
3798           s->bucket_name.c_str(),
3799           s->object.name.c_str()
3800           );
3801     }
3802     s->formatter->dump_string("Bucket", s->bucket_name);
3803     s->formatter->dump_string("Key", s->object.name);
3804     s->formatter->dump_string("ETag", etag);
3805     s->formatter->close_section();
3806     rgw_flush_formatter_and_reset(s, s->formatter);
3807   }
3808 }
3809
3810 void RGWAbortMultipart_ObjStore_S3::send_response()
3811 {
3812   int r = op_ret;
3813   if (!r)
3814     r = STATUS_NO_CONTENT;
3815
3816   set_req_state_err(s, r);
3817   dump_errno(s);
3818   end_header(s, this);
3819 }
3820
3821 void RGWListMultipart_ObjStore_S3::send_response()
3822 {
3823   if (op_ret)
3824     set_req_state_err(s, op_ret);
3825   dump_errno(s);
3826   // Explicitly use chunked transfer encoding so that we can stream the result
3827   // to the user without having to wait for the full length of it.
3828   end_header(s, this, "application/xml", CHUNKED_TRANSFER_ENCODING);
3829
3830   if (op_ret == 0) {
3831     dump_start(s);
3832     s->formatter->open_object_section_in_ns("ListPartsResult", XMLNS_AWS_S3);
3833     map<uint32_t, RGWUploadPartInfo>::iterator iter;
3834     map<uint32_t, RGWUploadPartInfo>::reverse_iterator test_iter;
3835     int cur_max = 0;
3836
3837     iter = parts.begin();
3838     test_iter = parts.rbegin();
3839     if (test_iter != parts.rend()) {
3840       cur_max = test_iter->first;
3841     }
3842     if (!s->bucket_tenant.empty())
3843       s->formatter->dump_string("Tenant", s->bucket_tenant);
3844     s->formatter->dump_string("Bucket", s->bucket_name);
3845     s->formatter->dump_string("Key", s->object.name);
3846     s->formatter->dump_string("UploadId", upload_id);
3847     s->formatter->dump_string("StorageClass", "STANDARD");
3848     s->formatter->dump_int("PartNumberMarker", marker);
3849     s->formatter->dump_int("NextPartNumberMarker", cur_max);
3850     s->formatter->dump_int("MaxParts", max_parts);
3851     s->formatter->dump_string("IsTruncated", (truncated ? "true" : "false"));
3852
3853     ACLOwner& owner = policy.get_owner();
3854     dump_owner(s, owner.get_id(), owner.get_display_name());
3855
3856     for (; iter != parts.end(); ++iter) {
3857       RGWUploadPartInfo& info = iter->second;
3858
3859       s->formatter->open_object_section("Part");
3860
3861       dump_time(s, "LastModified", &info.modified);
3862
3863       s->formatter->dump_unsigned("PartNumber", info.num);
3864       s->formatter->dump_format("ETag", "\"%s\"", info.etag.c_str());
3865       s->formatter->dump_unsigned("Size", info.accounted_size);
3866       s->formatter->close_section();
3867     }
3868     s->formatter->close_section();
3869     rgw_flush_formatter_and_reset(s, s->formatter);
3870   }
3871 }
3872
3873 void RGWListBucketMultiparts_ObjStore_S3::send_response()
3874 {
3875   if (op_ret < 0)
3876     set_req_state_err(s, op_ret);
3877   dump_errno(s);
3878
3879   // Explicitly use chunked transfer encoding so that we can stream the result
3880   // to the user without having to wait for the full length of it.
3881   end_header(s, this, "application/xml", CHUNKED_TRANSFER_ENCODING);
3882   dump_start(s);
3883   if (op_ret < 0)
3884     return;
3885
3886   s->formatter->open_object_section_in_ns("ListMultipartUploadsResult", XMLNS_AWS_S3);
3887   if (!s->bucket_tenant.empty())
3888     s->formatter->dump_string("Tenant", s->bucket_tenant);
3889   s->formatter->dump_string("Bucket", s->bucket_name);
3890   if (!prefix.empty())
3891     s->formatter->dump_string("ListMultipartUploadsResult.Prefix", prefix);
3892   const string& key_marker = marker.get_key();
3893   if (!key_marker.empty())
3894     s->formatter->dump_string("KeyMarker", key_marker);
3895   const string& upload_id_marker = marker.get_upload_id();
3896   if (!upload_id_marker.empty())
3897     s->formatter->dump_string("UploadIdMarker", upload_id_marker);
3898   string next_key = next_marker.mp.get_key();
3899   if (!next_key.empty())
3900     s->formatter->dump_string("NextKeyMarker", next_key);
3901   string next_upload_id = next_marker.mp.get_upload_id();
3902   if (!next_upload_id.empty())
3903     s->formatter->dump_string("NextUploadIdMarker", next_upload_id);
3904   s->formatter->dump_int("MaxUploads", max_uploads);
3905   if (!delimiter.empty())
3906     s->formatter->dump_string("Delimiter", delimiter);
3907   s->formatter->dump_string("IsTruncated", (is_truncated ? "true" : "false"));
3908
3909   if (op_ret >= 0) {
3910     vector<RGWMultipartUploadEntry>::iterator iter;
3911     for (iter = uploads.begin(); iter != uploads.end(); ++iter) {
3912       RGWMPObj& mp = iter->mp;
3913       s->formatter->open_array_section("Upload");
3914       if (encode_url) {
3915         s->formatter->dump_string("Key", url_encode(mp.get_key(), false));
3916       } else {
3917         s->formatter->dump_string("Key", mp.get_key());
3918       }
3919       s->formatter->dump_string("UploadId", mp.get_upload_id());
3920       dump_owner(s, s->user->get_id(), s->user->get_display_name(), "Initiator");
3921       dump_owner(s, s->user->get_id(), s->user->get_display_name());
3922       s->formatter->dump_string("StorageClass", "STANDARD");
3923       dump_time(s, "Initiated", &iter->obj.meta.mtime);
3924       s->formatter->close_section();
3925     }
3926     if (!common_prefixes.empty()) {
3927       s->formatter->open_array_section("CommonPrefixes");
3928       for (const auto& kv : common_prefixes) {
3929         if (encode_url) {
3930           s->formatter->dump_string("CommonPrefixes.Prefix",
3931                                     url_encode(kv.first, false));
3932         } else {
3933           s->formatter->dump_string("CommonPrefixes.Prefix", kv.first);
3934         }
3935       }
3936       s->formatter->close_section();
3937     }
3938   }
3939   s->formatter->close_section();
3940   rgw_flush_formatter_and_reset(s, s->formatter);
3941 }
3942
3943 int RGWDeleteMultiObj_ObjStore_S3::get_params()
3944 {
3945   int ret = RGWDeleteMultiObj_ObjStore::get_params();
3946   if (ret < 0) {
3947     return ret;
3948   }
3949
3950   return do_aws4_auth_completion();
3951 }
3952
3953 void RGWDeleteMultiObj_ObjStore_S3::send_status()
3954 {
3955   if (! status_dumped) {
3956     if (op_ret < 0)
3957       set_req_state_err(s, op_ret);
3958     dump_errno(s);
3959     status_dumped = true;
3960   }
3961 }
3962
3963 void RGWDeleteMultiObj_ObjStore_S3::begin_response()
3964 {
3965
3966   if (!status_dumped) {
3967     send_status();
3968   }
3969
3970   dump_start(s);
3971   // Explicitly use chunked transfer encoding so that we can stream the result
3972   // to the user without having to wait for the full length of it.
3973   end_header(s, this, "application/xml", CHUNKED_TRANSFER_ENCODING);
3974   s->formatter->open_object_section_in_ns("DeleteResult", XMLNS_AWS_S3);
3975
3976   rgw_flush_formatter(s, s->formatter);
3977 }
3978
3979 void RGWDeleteMultiObj_ObjStore_S3::send_partial_response(rgw_obj_key& key,
3980                                                           bool delete_marker,
3981                                                           const string& marker_version_id, int ret)
3982 {
3983   if (!key.empty()) {
3984     if (ret == 0 && !quiet) {
3985       s->formatter->open_object_section("Deleted");
3986       s->formatter->dump_string("Key", key.name);
3987       if (!key.instance.empty()) {
3988         s->formatter->dump_string("VersionId", key.instance);
3989       }
3990       if (delete_marker) {
3991         s->formatter->dump_bool("DeleteMarker", true);
3992         s->formatter->dump_string("DeleteMarkerVersionId", marker_version_id);
3993       }
3994       s->formatter->close_section();
3995     } else if (ret < 0) {
3996       struct rgw_http_error r;
3997       int err_no;
3998
3999       s->formatter->open_object_section("Error");
4000
4001       err_no = -ret;
4002       rgw_get_errno_s3(&r, err_no);
4003
4004       s->formatter->dump_string("Key", key.name);
4005       s->formatter->dump_string("VersionId", key.instance);
4006       s->formatter->dump_string("Code", r.s3_code);
4007       s->formatter->dump_string("Message", r.s3_code);
4008       s->formatter->close_section();
4009     }
4010
4011     rgw_flush_formatter(s, s->formatter);
4012   }
4013 }
4014
4015 void RGWDeleteMultiObj_ObjStore_S3::end_response()
4016 {
4017
4018   s->formatter->close_section();
4019   rgw_flush_formatter_and_reset(s, s->formatter);
4020 }
4021
4022 void RGWGetObjLayout_ObjStore_S3::send_response()
4023 {
4024   if (op_ret)
4025     set_req_state_err(s, op_ret);
4026   dump_errno(s);
4027   end_header(s, this, "application/json");
4028
4029   JSONFormatter f;
4030
4031   if (op_ret < 0) {
4032     return;
4033   }
4034
4035   f.open_object_section("result");
4036   ::encode_json("head", head_obj, &f);
4037   ::encode_json("manifest", *manifest, &f);
4038   f.open_array_section("data_location");
4039   for (auto miter = manifest->obj_begin(); miter != manifest->obj_end(); ++miter) {
4040     f.open_object_section("obj");
4041     rgw_raw_obj raw_loc = miter.get_location().get_raw_obj(store->getRados());
4042     uint64_t ofs = miter.get_ofs();
4043     uint64_t left = manifest->get_obj_size() - ofs;
4044     ::encode_json("ofs", miter.get_ofs(), &f);
4045     ::encode_json("loc", raw_loc, &f);
4046     ::encode_json("loc_ofs", miter.location_ofs(), &f);
4047     uint64_t loc_size = miter.get_stripe_size();
4048     if (loc_size > left) {
4049       loc_size = left;
4050     }
4051     ::encode_json("loc_size", loc_size, &f);
4052     f.close_section();
4053     rgw_flush_formatter(s, &f);
4054   }
4055   f.close_section();
4056   f.close_section();
4057   rgw_flush_formatter(s, &f);
4058 }
4059
4060 int RGWConfigBucketMetaSearch_ObjStore_S3::get_params()
4061 {
4062   auto iter = s->info.x_meta_map.find("x-amz-meta-search");
4063   if (iter == s->info.x_meta_map.end()) {
4064     s->err.message = "X-Rgw-Meta-Search header not provided";
4065     ldpp_dout(this, 5) << s->err.message << dendl;
4066     return -EINVAL;
4067   }
4068
4069   list<string> expressions;
4070   get_str_list(iter->second, ",", expressions);
4071
4072   for (auto& expression : expressions) {
4073     vector<string> args;
4074     get_str_vec(expression, ";", args);
4075
4076     if (args.empty()) {
4077       s->err.message = "invalid empty expression";
4078       ldpp_dout(this, 5) << s->err.message << dendl;
4079       return -EINVAL;
4080     }
4081     if (args.size() > 2) {
4082       s->err.message = string("invalid expression: ") + expression;
4083       ldpp_dout(this, 5) << s->err.message << dendl;
4084       return -EINVAL;
4085     }
4086
4087     string key = boost::algorithm::to_lower_copy(rgw_trim_whitespace(args[0]));
4088     string val;
4089     if (args.size() > 1) {
4090       val = boost::algorithm::to_lower_copy(rgw_trim_whitespace(args[1]));
4091     }
4092
4093     if (!boost::algorithm::starts_with(key, RGW_AMZ_META_PREFIX)) {
4094       s->err.message = string("invalid expression, key must start with '" RGW_AMZ_META_PREFIX "' : ") + expression;
4095       ldpp_dout(this, 5) << s->err.message << dendl;
4096       return -EINVAL;
4097     }
4098
4099     key = key.substr(sizeof(RGW_AMZ_META_PREFIX) - 1);
4100
4101     ESEntityTypeMap::EntityType entity_type;
4102
4103     if (val.empty() || val == "str" || val == "string") {
4104       entity_type = ESEntityTypeMap::ES_ENTITY_STR;
4105     } else if (val == "int" || val == "integer") {
4106       entity_type = ESEntityTypeMap::ES_ENTITY_INT;
4107     } else if (val == "date" || val == "datetime") {
4108       entity_type = ESEntityTypeMap::ES_ENTITY_DATE;
4109     } else {
4110       s->err.message = string("invalid entity type: ") + val;
4111       ldpp_dout(this, 5) << s->err.message << dendl;
4112       return -EINVAL;
4113     }
4114
4115     mdsearch_config[key] = entity_type;
4116   }
4117
4118   return 0;
4119 }
4120
4121 void RGWConfigBucketMetaSearch_ObjStore_S3::send_response()
4122 {
4123   if (op_ret)
4124     set_req_state_err(s, op_ret);
4125   dump_errno(s);
4126   end_header(s, this);
4127 }
4128
4129 void RGWGetBucketMetaSearch_ObjStore_S3::send_response()
4130 {
4131   if (op_ret)
4132     set_req_state_err(s, op_ret);
4133   dump_errno(s);
4134   end_header(s, NULL, "application/xml");
4135
4136   Formatter *f = s->formatter;
4137   f->open_array_section("GetBucketMetaSearchResult");
4138   for (auto& e : s->bucket_info.mdsearch_config) {
4139     f->open_object_section("Entry");
4140     string k = string("x-amz-meta-") + e.first;
4141     f->dump_string("Key", k.c_str());
4142     const char *type;
4143     switch (e.second) {
4144       case ESEntityTypeMap::ES_ENTITY_INT:
4145         type = "int";
4146         break;
4147       case ESEntityTypeMap::ES_ENTITY_DATE:
4148         type = "date";
4149         break;
4150       default:
4151         type = "str";
4152     }
4153     f->dump_string("Type", type);
4154     f->close_section();
4155   }
4156   f->close_section();
4157   rgw_flush_formatter(s, f);
4158 }
4159
4160 void RGWDelBucketMetaSearch_ObjStore_S3::send_response()
4161 {
4162   if (op_ret)
4163     set_req_state_err(s, op_ret);
4164   dump_errno(s);
4165   end_header(s, this);
4166 }
4167
4168 void RGWPutBucketObjectLock_ObjStore_S3::send_response()
4169 {
4170   if (op_ret) {
4171     set_req_state_err(s, op_ret);
4172   }
4173   dump_errno(s);
4174   end_header(s);
4175 }
4176
4177 void RGWGetBucketObjectLock_ObjStore_S3::send_response()
4178 {
4179   if (op_ret) {
4180     set_req_state_err(s, op_ret);
4181   }
4182   dump_errno(s);
4183   end_header(s, this, "application/xml");
4184   dump_start(s);
4185
4186   if (op_ret) {
4187     return;
4188   }
4189   encode_xml("ObjectLockConfiguration", s->bucket_info.obj_lock, s->formatter);
4190   rgw_flush_formatter_and_reset(s, s->formatter);
4191 }
4192
4193
4194 int RGWPutObjRetention_ObjStore_S3::get_params()
4195 {
4196   const char *bypass_gov_header = s->info.env->get("HTTP_X_AMZ_BYPASS_GOVERNANCE_RETENTION");
4197   if (bypass_gov_header) {
4198     std::string bypass_gov_decoded = url_decode(bypass_gov_header);
4199     bypass_governance_mode = boost::algorithm::iequals(bypass_gov_decoded, "true");
4200   }
4201
4202   const auto max_size = s->cct->_conf->rgw_max_put_param_size;
4203   std::tie(op_ret, data) = rgw_rest_read_all_input(s, max_size, false);
4204   return op_ret;
4205 }
4206
4207 void RGWPutObjRetention_ObjStore_S3::send_response()
4208 {
4209   if (op_ret) {
4210     set_req_state_err(s, op_ret);
4211   }
4212   dump_errno(s);
4213   end_header(s);
4214 }
4215
4216 void RGWGetObjRetention_ObjStore_S3::send_response()
4217 {
4218   if (op_ret) {
4219     set_req_state_err(s, op_ret);
4220   }
4221   dump_errno(s);
4222   end_header(s, this, "application/xml");
4223   dump_start(s);
4224
4225   if (op_ret) {
4226     return;
4227   }
4228   encode_xml("Retention", obj_retention, s->formatter);
4229   rgw_flush_formatter_and_reset(s, s->formatter);
4230 }
4231
4232 void RGWPutObjLegalHold_ObjStore_S3::send_response()
4233 {
4234   if (op_ret) {
4235     set_req_state_err(s, op_ret);
4236   }
4237   dump_errno(s);
4238   end_header(s);
4239 }
4240
4241 void RGWGetObjLegalHold_ObjStore_S3::send_response()
4242 {
4243   if (op_ret) {
4244     set_req_state_err(s, op_ret);
4245   }
4246   dump_errno(s);
4247   end_header(s, this, "application/xml");
4248   dump_start(s);
4249
4250   if (op_ret) {
4251     return;
4252   }
4253   encode_xml("LegalHold", obj_legal_hold, s->formatter);
4254   rgw_flush_formatter_and_reset(s, s->formatter);
4255 }
4256
4257 void RGWGetBucketPolicyStatus_ObjStore_S3::send_response()
4258 {
4259   if (op_ret) {
4260     set_req_state_err(s, op_ret);
4261   }
4262   dump_errno(s);
4263   end_header(s, this, "application/xml");
4264   dump_start(s);
4265
4266   s->formatter->open_object_section_in_ns("PolicyStatus", XMLNS_AWS_S3);
4267   // https://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGETPolicyStatus.html
4268   // mentions TRUE and FALSE, but boto/aws official clients seem to want lower
4269   // case which is returned by AWS as well; so let's be bug to bug compatible
4270   // with the API
4271   s->formatter->dump_bool("IsPublic", isPublic);
4272   s->formatter->close_section();
4273   rgw_flush_formatter_and_reset(s, s->formatter);
4274
4275 }
4276
4277 void RGWPutBucketPublicAccessBlock_ObjStore_S3::send_response()
4278 {
4279   if (op_ret) {
4280     set_req_state_err(s, op_ret);
4281   }
4282   dump_errno(s);
4283   end_header(s);
4284 }
4285
4286 void RGWGetBucketPublicAccessBlock_ObjStore_S3::send_response()
4287 {
4288   if (op_ret) {
4289     set_req_state_err(s, op_ret);
4290   }
4291   dump_errno(s);
4292   end_header(s, this, "application/xml");
4293   dump_start(s);
4294
4295   access_conf.dump_xml(s->formatter);
4296   rgw_flush_formatter_and_reset(s, s->formatter);
4297 }
4298
4299 RGWOp *RGWHandler_REST_Service_S3::op_get()
4300 {
4301   if (is_usage_op()) {
4302     return new RGWGetUsage_ObjStore_S3;
4303   } else {
4304     return new RGWListBuckets_ObjStore_S3;
4305   }
4306 }
4307
4308 RGWOp *RGWHandler_REST_Service_S3::op_head()
4309 {
4310   return new RGWListBuckets_ObjStore_S3;
4311 }
4312
4313 RGWOp *RGWHandler_REST_Service_S3::op_post()
4314 {
4315   const auto max_size = s->cct->_conf->rgw_max_put_param_size;
4316
4317   int ret;
4318   bufferlist data;
4319   std::tie(ret, data) = rgw_rest_read_all_input(s, max_size, false);
4320   if (ret < 0) {
4321       return nullptr;
4322   }
4323
4324   const auto post_body = data.to_str();
4325
4326   if (isSTSEnabled) {
4327     RGWHandler_REST_STS sts_handler(auth_registry, post_body);
4328     sts_handler.init(store, s, s->cio);
4329     auto op = sts_handler.get_op();
4330     if (op) {
4331       return op;
4332     }
4333   }
4334
4335   if (isIAMEnabled) {
4336     RGWHandler_REST_IAM iam_handler(auth_registry, post_body);
4337     iam_handler.init(store, s, s->cio);
4338     auto op = iam_handler.get_op();
4339     if (op) {
4340       return op;
4341     }
4342   }
4343
4344   if (isPSEnabled) {
4345     RGWHandler_REST_PSTopic_AWS topic_handler(auth_registry, post_body);
4346     topic_handler.init(store, s, s->cio);
4347     auto op = topic_handler.get_op();
4348     if (op) {
4349       return op;
4350     }
4351   }
4352
4353   return nullptr;
4354 }
4355
4356 RGWOp *RGWHandler_REST_Bucket_S3::get_obj_op(bool get_data) const
4357 {
4358   // Non-website mode
4359   if (get_data) {
4360     int list_type = 1;
4361     s->info.args.get_int("list-type", &list_type, 1);
4362     switch (list_type) {
4363       case 1:
4364         return new RGWListBucket_ObjStore_S3;
4365       case 2:
4366         return new RGWListBucket_ObjStore_S3v2;
4367       default:
4368         ldpp_dout(s, 5) << __func__ << ": unsupported list-type " << list_type << dendl;
4369         return new RGWListBucket_ObjStore_S3;
4370     }
4371   } else {
4372     return new RGWStatBucket_ObjStore_S3;
4373   }
4374 }
4375
4376 RGWOp *RGWHandler_REST_Bucket_S3::op_get()
4377 {
4378   if (s->info.args.sub_resource_exists("encryption"))
4379     return nullptr;
4380
4381   if (s->info.args.sub_resource_exists("logging"))
4382     return new RGWGetBucketLogging_ObjStore_S3;
4383
4384   if (s->info.args.sub_resource_exists("location"))
4385     return new RGWGetBucketLocation_ObjStore_S3;
4386
4387   if (s->info.args.sub_resource_exists("versioning"))
4388     return new RGWGetBucketVersioning_ObjStore_S3;
4389
4390   if (s->info.args.sub_resource_exists("website")) {
4391     if (!s->cct->_conf->rgw_enable_static_website) {
4392       return NULL;
4393     }
4394     return new RGWGetBucketWebsite_ObjStore_S3;
4395   }
4396
4397   if (s->info.args.exists("mdsearch")) {
4398     return new RGWGetBucketMetaSearch_ObjStore_S3;
4399   }
4400
4401   if (is_acl_op()) {
4402     return new RGWGetACLs_ObjStore_S3;
4403   } else if (is_cors_op()) {
4404     return new RGWGetCORS_ObjStore_S3;
4405   } else if (is_request_payment_op()) {
4406     return new RGWGetRequestPayment_ObjStore_S3;
4407   } else if (s->info.args.exists("uploads")) {
4408     return new RGWListBucketMultiparts_ObjStore_S3;
4409   } else if(is_lc_op()) {
4410     return new RGWGetLC_ObjStore_S3;
4411   } else if(is_policy_op()) {
4412     return new RGWGetBucketPolicy;
4413   } else if (is_tagging_op()) {
4414     return new RGWGetBucketTags_ObjStore_S3;
4415   } else if (is_object_lock_op()) {
4416     return new RGWGetBucketObjectLock_ObjStore_S3;
4417   } else if (is_notification_op()) {
4418     return RGWHandler_REST_PSNotifs_S3::create_get_op();
4419   } else if (is_replication_op()) {
4420     return new RGWGetBucketReplication_ObjStore_S3;
4421   } else if (is_policy_status_op()) {
4422     return new RGWGetBucketPolicyStatus_ObjStore_S3;
4423   } else if (is_block_public_access_op()) {
4424     return new RGWGetBucketPublicAccessBlock_ObjStore_S3;
4425   }
4426   return get_obj_op(true);
4427 }
4428
4429 RGWOp *RGWHandler_REST_Bucket_S3::op_head()
4430 {
4431   if (is_acl_op()) {
4432     return new RGWGetACLs_ObjStore_S3;
4433   } else if (s->info.args.exists("uploads")) {
4434     return new RGWListBucketMultiparts_ObjStore_S3;
4435   }
4436   return get_obj_op(false);
4437 }
4438
4439 RGWOp *RGWHandler_REST_Bucket_S3::op_put()
4440 {
4441   if (s->info.args.sub_resource_exists("logging") ||
4442       s->info.args.sub_resource_exists("encryption"))
4443     return nullptr;
4444   if (s->info.args.sub_resource_exists("versioning"))
4445     return new RGWSetBucketVersioning_ObjStore_S3;
4446   if (s->info.args.sub_resource_exists("website")) {
4447     if (!s->cct->_conf->rgw_enable_static_website) {
4448       return NULL;
4449     }
4450     return new RGWSetBucketWebsite_ObjStore_S3;
4451   }
4452   if (is_tagging_op()) {
4453     return new RGWPutBucketTags_ObjStore_S3;
4454   } else if (is_acl_op()) {
4455     return new RGWPutACLs_ObjStore_S3;
4456   } else if (is_cors_op()) {
4457     return new RGWPutCORS_ObjStore_S3;
4458   } else if (is_request_payment_op()) {
4459     return new RGWSetRequestPayment_ObjStore_S3;
4460   } else if(is_lc_op()) {
4461     return new RGWPutLC_ObjStore_S3;
4462   } else if(is_policy_op()) {
4463     return new RGWPutBucketPolicy;
4464   } else if (is_object_lock_op()) {
4465     return new RGWPutBucketObjectLock_ObjStore_S3;
4466   } else if (is_notification_op()) {
4467     return RGWHandler_REST_PSNotifs_S3::create_put_op();
4468   } else if (is_replication_op()) {
4469     auto sync_policy_handler = store->svc()->zone->get_sync_policy_handler(nullopt);
4470     if (!sync_policy_handler ||
4471         sync_policy_handler->is_legacy_config()) {
4472       return nullptr;
4473     }
4474
4475     return new RGWPutBucketReplication_ObjStore_S3;
4476   } else if (is_block_public_access_op()) {
4477     return new RGWPutBucketPublicAccessBlock_ObjStore_S3;
4478   }
4479   return new RGWCreateBucket_ObjStore_S3;
4480 }
4481
4482 RGWOp *RGWHandler_REST_Bucket_S3::op_delete()
4483 {
4484   if (s->info.args.sub_resource_exists("logging") ||
4485       s->info.args.sub_resource_exists("encryption"))
4486     return nullptr;
4487
4488   if (is_tagging_op()) {
4489     return new RGWDeleteBucketTags_ObjStore_S3;
4490   } else if (is_cors_op()) {
4491     return new RGWDeleteCORS_ObjStore_S3;
4492   } else if(is_lc_op()) {
4493     return new RGWDeleteLC_ObjStore_S3;
4494   } else if(is_policy_op()) {
4495     return new RGWDeleteBucketPolicy;
4496   } else if (is_notification_op()) {
4497     return RGWHandler_REST_PSNotifs_S3::create_delete_op();
4498   } else if (is_replication_op()) {
4499     return new RGWDeleteBucketReplication_ObjStore_S3;
4500   } else if (is_block_public_access_op()) {
4501     return new RGWDeleteBucketPublicAccessBlock;
4502   }
4503
4504   if (s->info.args.sub_resource_exists("website")) {
4505     if (!s->cct->_conf->rgw_enable_static_website) {
4506       return NULL;
4507     }
4508     return new RGWDeleteBucketWebsite_ObjStore_S3;
4509   }
4510
4511   if (s->info.args.exists("mdsearch")) {
4512     return new RGWDelBucketMetaSearch_ObjStore_S3;
4513   }
4514
4515   return new RGWDeleteBucket_ObjStore_S3;
4516 }
4517
4518 RGWOp *RGWHandler_REST_Bucket_S3::op_post()
4519 {
4520   if (s->info.args.exists("delete")) {
4521     return new RGWDeleteMultiObj_ObjStore_S3;
4522   }
4523
4524   if (s->info.args.exists("mdsearch")) {
4525     return new RGWConfigBucketMetaSearch_ObjStore_S3;
4526   }
4527
4528   return new RGWPostObj_ObjStore_S3;
4529 }
4530
4531 RGWOp *RGWHandler_REST_Bucket_S3::op_options()
4532 {
4533   return new RGWOptionsCORS_ObjStore_S3;
4534 }
4535
4536 RGWOp *RGWHandler_REST_Obj_S3::get_obj_op(bool get_data)
4537 {
4538   RGWGetObj_ObjStore_S3 *get_obj_op = new RGWGetObj_ObjStore_S3;
4539   get_obj_op->set_get_data(get_data);
4540   return get_obj_op;
4541 }
4542
4543 RGWOp *RGWHandler_REST_Obj_S3::op_get()
4544 {
4545   if (is_acl_op()) {
4546     return new RGWGetACLs_ObjStore_S3;
4547   } else if (s->info.args.exists("uploadId")) {
4548     return new RGWListMultipart_ObjStore_S3;
4549   } else if (s->info.args.exists("layout")) {
4550     return new RGWGetObjLayout_ObjStore_S3;
4551   } else if (is_tagging_op()) {
4552     return new RGWGetObjTags_ObjStore_S3;
4553   } else if (is_obj_retention_op()) {
4554     return new RGWGetObjRetention_ObjStore_S3;
4555   } else if (is_obj_legal_hold_op()) {
4556     return new RGWGetObjLegalHold_ObjStore_S3;
4557   }
4558   return get_obj_op(true);
4559 }
4560
4561 RGWOp *RGWHandler_REST_Obj_S3::op_head()
4562 {
4563   if (is_acl_op()) {
4564     return new RGWGetACLs_ObjStore_S3;
4565   } else if (s->info.args.exists("uploadId")) {
4566     return new RGWListMultipart_ObjStore_S3;
4567   }
4568   return get_obj_op(false);
4569 }
4570
4571 RGWOp *RGWHandler_REST_Obj_S3::op_put()
4572 {
4573   if (is_acl_op()) {
4574     return new RGWPutACLs_ObjStore_S3;
4575   } else if (is_tagging_op()) {
4576     return new RGWPutObjTags_ObjStore_S3;
4577   } else if (is_obj_retention_op()) {
4578     return new RGWPutObjRetention_ObjStore_S3;
4579   } else if (is_obj_legal_hold_op()) {
4580     return new RGWPutObjLegalHold_ObjStore_S3;
4581   }
4582
4583   if (s->init_state.src_bucket.empty())
4584     return new RGWPutObj_ObjStore_S3;
4585   else
4586     return new RGWCopyObj_ObjStore_S3;
4587 }
4588
4589 RGWOp *RGWHandler_REST_Obj_S3::op_delete()
4590 {
4591   if (is_tagging_op()) {
4592     return new RGWDeleteObjTags_ObjStore_S3;
4593   }
4594   string upload_id = s->info.args.get("uploadId");
4595
4596   if (upload_id.empty())
4597     return new RGWDeleteObj_ObjStore_S3;
4598   else
4599     return new RGWAbortMultipart_ObjStore_S3;
4600 }
4601
4602 RGWOp *RGWHandler_REST_Obj_S3::op_post()
4603 {
4604   if (s->info.args.exists("uploadId"))
4605     return new RGWCompleteMultipart_ObjStore_S3;
4606
4607   if (s->info.args.exists("uploads"))
4608     return new RGWInitMultipart_ObjStore_S3;
4609
4610   return new RGWPostObj_ObjStore_S3;
4611 }
4612
4613 RGWOp *RGWHandler_REST_Obj_S3::op_options()
4614 {
4615   return new RGWOptionsCORS_ObjStore_S3;
4616 }
4617
4618 int RGWHandler_REST_S3::init_from_header(struct req_state* s,
4619                                         int default_formatter,
4620                                         bool configurable_format)
4621 {
4622   string req;
4623   string first;
4624
4625   const char *req_name = s->relative_uri.c_str();
4626   const char *p;
4627
4628   if (*req_name == '?') {
4629     p = req_name;
4630   } else {
4631     p = s->info.request_params.c_str();
4632   }
4633
4634   s->info.args.set(p);
4635   s->info.args.parse();
4636
4637   /* must be called after the args parsing */
4638   int ret = allocate_formatter(s, default_formatter, configurable_format);
4639   if (ret < 0)
4640     return ret;
4641
4642   if (*req_name != '/')
4643     return 0;
4644
4645   req_name++;
4646
4647   if (!*req_name)
4648     return 0;
4649
4650   req = req_name;
4651   int pos = req.find('/');
4652   if (pos >= 0) {
4653     first = req.substr(0, pos);
4654   } else {
4655     first = req;
4656   }
4657
4658   /*
4659    * XXX The intent of the check for empty is apparently to let the bucket
4660    * name from DNS to be set ahead. However, we currently take the DNS
4661    * bucket and re-insert it into URL in rgw_rest.cc:RGWREST::preprocess().
4662    * So, this check is meaningless.
4663    *
4664    * Rather than dropping this, the code needs to be changed into putting
4665    * the bucket (and its tenant) from DNS and Host: header (HTTP_HOST)
4666    * into req_status.bucket_name directly.
4667    */
4668   if (s->init_state.url_bucket.empty()) {
4669     // Save bucket to tide us over until token is parsed.
4670     s->init_state.url_bucket = first;
4671     if (pos >= 0) {
4672       string encoded_obj_str = req.substr(pos+1);
4673       s->object = rgw_obj_key(encoded_obj_str, s->info.args.get("versionId"));
4674     }
4675   } else {
4676     s->object = rgw_obj_key(req_name, s->info.args.get("versionId"));
4677   }
4678   return 0;
4679 }
4680
4681 static int verify_mfa(rgw::sal::RGWRadosStore *store, RGWUserInfo *user, const string& mfa_str, bool *verified, const DoutPrefixProvider *dpp)
4682 {
4683   vector<string> params;
4684   get_str_vec(mfa_str, " ", params);
4685
4686   if (params.size() != 2) {
4687     ldpp_dout(dpp, 5) << "NOTICE: invalid mfa string provided: " << mfa_str << dendl;
4688     return -EINVAL;
4689   }
4690
4691   string& serial = params[0];
4692   string& pin = params[1];
4693
4694   auto i = user->mfa_ids.find(serial);
4695   if (i == user->mfa_ids.end()) {
4696     ldpp_dout(dpp, 5) << "NOTICE: user does not have mfa device with serial=" << serial << dendl;
4697     return -EACCES;
4698   }
4699
4700   int ret = store->svc()->cls->mfa.check_mfa(user->user_id, serial, pin, null_yield);
4701   if (ret < 0) {
4702     ldpp_dout(dpp, 20) << "NOTICE: failed to check MFA, serial=" << serial << dendl;
4703     return -EACCES;
4704   }
4705
4706   *verified = true;
4707
4708   return 0;
4709 }
4710
4711 int RGWHandler_REST_S3::postauth_init()
4712 {
4713   struct req_init_state *t = &s->init_state;
4714
4715   rgw_parse_url_bucket(t->url_bucket, s->user->get_tenant(),
4716                       s->bucket_tenant, s->bucket_name);
4717
4718   dout(10) << "s->object=" << (!s->object.empty() ? s->object : rgw_obj_key("<NULL>"))
4719            << " s->bucket=" << rgw_make_bucket_entry_name(s->bucket_tenant, s->bucket_name) << dendl;
4720
4721   int ret;
4722   ret = rgw_validate_tenant_name(s->bucket_tenant);
4723   if (ret)
4724     return ret;
4725   if (!s->bucket_name.empty()) {
4726     ret = validate_object_name(s->object.name);
4727     if (ret)
4728       return ret;
4729   }
4730
4731   if (!t->src_bucket.empty()) {
4732     rgw_parse_url_bucket(t->src_bucket, s->user->get_tenant(),
4733                         s->src_tenant_name, s->src_bucket_name);
4734     ret = rgw_validate_tenant_name(s->src_tenant_name);
4735     if (ret)
4736       return ret;
4737   }
4738
4739   const char *mfa = s->info.env->get("HTTP_X_AMZ_MFA");
4740   if (mfa) {
4741     ret = verify_mfa(store, &s->user->get_info(), string(mfa), &s->mfa_verified, s);
4742   }
4743
4744   return 0;
4745 }
4746
4747 int RGWHandler_REST_S3::init(rgw::sal::RGWRadosStore *store, struct req_state *s,
4748                              rgw::io::BasicClient *cio)
4749 {
4750   int ret;
4751
4752   s->dialect = "s3";
4753
4754   ret = rgw_validate_tenant_name(s->bucket_tenant);
4755   if (ret)
4756     return ret;
4757   if (!s->bucket_name.empty()) {
4758     ret = validate_object_name(s->object.name);
4759     if (ret)
4760       return ret;
4761   }
4762
4763   const char *cacl = s->info.env->get("HTTP_X_AMZ_ACL");
4764   if (cacl)
4765     s->canned_acl = cacl;
4766
4767   s->has_acl_header = s->info.env->exists_prefix("HTTP_X_AMZ_GRANT");
4768
4769   const char *copy_source = s->info.env->get("HTTP_X_AMZ_COPY_SOURCE");
4770   if (copy_source &&
4771       (! s->info.env->get("HTTP_X_AMZ_COPY_SOURCE_RANGE")) &&
4772       (! s->info.args.exists("uploadId"))) {
4773
4774     ret = RGWCopyObj::parse_copy_location(copy_source,
4775                                           s->init_state.src_bucket,
4776                                           s->src_object);
4777     if (!ret) {
4778       ldpp_dout(s, 0) << "failed to parse copy location" << dendl;
4779       return -EINVAL; // XXX why not -ERR_INVALID_BUCKET_NAME or -ERR_BAD_URL?
4780     }
4781   }
4782
4783   const char *sc = s->info.env->get("HTTP_X_AMZ_STORAGE_CLASS");
4784   if (sc) {
4785     s->info.storage_class = sc;
4786   }
4787
4788   return RGWHandler_REST::init(store, s, cio);
4789 }
4790
4791 int RGWHandler_REST_S3::authorize(const DoutPrefixProvider *dpp)
4792 {
4793   if (s->info.args.exists("Action") && s->info.args.get("Action") == "AssumeRoleWithWebIdentity") {
4794     return RGW_Auth_STS::authorize(dpp, store, auth_registry, s);
4795   }
4796   return RGW_Auth_S3::authorize(dpp, store, auth_registry, s);
4797 }
4798
4799 enum class AwsVersion {
4800   UNKNOWN,
4801   V2,
4802   V4
4803 };
4804
4805 enum class AwsRoute {
4806   UNKNOWN,
4807   QUERY_STRING,
4808   HEADERS
4809 };
4810
4811 static inline std::pair<AwsVersion, AwsRoute>
4812 discover_aws_flavour(const req_info& info)
4813 {
4814   using rgw::auth::s3::AWS4_HMAC_SHA256_STR;
4815
4816   AwsVersion version = AwsVersion::UNKNOWN;
4817   AwsRoute route = AwsRoute::UNKNOWN;
4818
4819   const char* http_auth = info.env->get("HTTP_AUTHORIZATION");
4820   if (http_auth && http_auth[0]) {
4821     /* Authorization in Header */
4822     route = AwsRoute::HEADERS;
4823
4824     if (!strncmp(http_auth, AWS4_HMAC_SHA256_STR,
4825                  strlen(AWS4_HMAC_SHA256_STR))) {
4826       /* AWS v4 */
4827       version = AwsVersion::V4;
4828     } else if (!strncmp(http_auth, "AWS ", 4)) {
4829       /* AWS v2 */
4830       version = AwsVersion::V2;
4831     }
4832   } else {
4833     route = AwsRoute::QUERY_STRING;
4834
4835     if (info.args.get("X-Amz-Algorithm") == AWS4_HMAC_SHA256_STR) {
4836       /* AWS v4 */
4837       version = AwsVersion::V4;
4838     } else if (!info.args.get("AWSAccessKeyId").empty()) {
4839       /* AWS v2 */
4840       version = AwsVersion::V2;
4841     }
4842   }
4843
4844   return std::make_pair(version, route);
4845 }
4846
4847 /*
4848  * verify that a signed request comes from the keyholder
4849  * by checking the signature against our locally-computed version
4850  *
4851  * it tries AWS v4 before AWS v2
4852  */
4853 int RGW_Auth_S3::authorize(const DoutPrefixProvider *dpp,
4854                            rgw::sal::RGWRadosStore* const store,
4855                            const rgw::auth::StrategyRegistry& auth_registry,
4856                            struct req_state* const s)
4857 {
4858
4859   /* neither keystone and rados enabled; warn and exit! */
4860   if (!store->ctx()->_conf->rgw_s3_auth_use_rados &&
4861       !store->ctx()->_conf->rgw_s3_auth_use_keystone &&
4862       !store->ctx()->_conf->rgw_s3_auth_use_ldap) {
4863     ldpp_dout(dpp, 0) << "WARNING: no authorization backend enabled! Users will never authenticate." << dendl;
4864     return -EPERM;
4865   }
4866
4867   const auto ret = rgw::auth::Strategy::apply(dpp, auth_registry.get_s3_main(), s);
4868   if (ret == 0) {
4869     /* Populate the owner info. */
4870     s->owner.set_id(s->user->get_id());
4871     s->owner.set_name(s->user->get_display_name());
4872   }
4873   return ret;
4874 }
4875
4876 int RGWHandler_Auth_S3::init(rgw::sal::RGWRadosStore *store, struct req_state *state,
4877                              rgw::io::BasicClient *cio)
4878 {
4879   int ret = RGWHandler_REST_S3::init_from_header(state, RGW_FORMAT_JSON,
4880                                                      true);
4881   if (ret < 0)
4882     return ret;
4883
4884   return RGWHandler_REST::init(store, state, cio);
4885 }
4886
4887 RGWHandler_REST* RGWRESTMgr_S3::get_handler(struct req_state* const s,
4888                                             const rgw::auth::StrategyRegistry& auth_registry,
4889                                             const std::string& frontend_prefix)
4890 {
4891   bool is_s3website = enable_s3website && (s->prot_flags & RGW_REST_WEBSITE);
4892   int ret =
4893     RGWHandler_REST_S3::init_from_header(s,
4894                                         is_s3website ? RGW_FORMAT_HTML :
4895                                         RGW_FORMAT_XML, true);
4896   if (ret < 0)
4897     return NULL;
4898
4899   RGWHandler_REST* handler;
4900   // TODO: Make this more readable
4901   if (is_s3website) {
4902     if (s->init_state.url_bucket.empty()) {
4903       handler = new RGWHandler_REST_Service_S3Website(auth_registry);
4904     } else if (s->object.empty()) {
4905       handler = new RGWHandler_REST_Bucket_S3Website(auth_registry);
4906     } else {
4907       handler = new RGWHandler_REST_Obj_S3Website(auth_registry);
4908     }
4909   } else {
4910     if (s->init_state.url_bucket.empty()) {
4911       handler = new RGWHandler_REST_Service_S3(auth_registry, enable_sts, enable_iam, enable_pubsub);
4912     } else if (s->object.empty()) {
4913       handler = new RGWHandler_REST_Bucket_S3(auth_registry, enable_pubsub);
4914     } else {
4915       handler = new RGWHandler_REST_Obj_S3(auth_registry);
4916     }
4917   }
4918
4919   ldpp_dout(s, 20) << __func__ << " handler=" << typeid(*handler).name()
4920                     << dendl;
4921   return handler;
4922 }
4923
4924 bool RGWHandler_REST_S3Website::web_dir() const {
4925   std::string subdir_name = url_decode(s->object.name);
4926
4927   if (subdir_name.empty()) {
4928     return false;
4929   } else if (subdir_name.back() == '/' && subdir_name.size() > 1) {
4930     subdir_name.pop_back();
4931   }
4932
4933   rgw_obj obj(s->bucket, subdir_name);
4934
4935   RGWObjectCtx& obj_ctx = *static_cast<RGWObjectCtx *>(s->obj_ctx);
4936   obj_ctx.set_atomic(obj);
4937   obj_ctx.set_prefetch_data(obj);
4938
4939   RGWObjState* state = nullptr;
4940   if (store->getRados()->get_obj_state(&obj_ctx, s->bucket_info, obj, &state, false, s->yield) < 0) {
4941     return false;
4942   }
4943   if (! state->exists) {
4944     return false;
4945   }
4946   return state->exists;
4947 }
4948
4949 int RGWHandler_REST_S3Website::init(rgw::sal::RGWRadosStore *store, req_state *s,
4950                                     rgw::io::BasicClient* cio)
4951 {
4952   // save the original object name before retarget() replaces it with the
4953   // result of get_effective_key(). the error_handler() needs the original
4954   // object name for redirect handling
4955   original_object_name = s->object.name;
4956
4957   return RGWHandler_REST_S3::init(store, s, cio);
4958 }
4959
4960 int RGWHandler_REST_S3Website::retarget(RGWOp* op, RGWOp** new_op) {
4961   *new_op = op;
4962   ldpp_dout(s, 10) << __func__ << " Starting retarget" << dendl;
4963
4964   if (!(s->prot_flags & RGW_REST_WEBSITE))
4965     return 0;
4966
4967   int ret = store->getRados()->get_bucket_info(store->svc(), s->bucket_tenant,
4968                                   s->bucket_name, s->bucket_info, NULL,
4969                                   s->yield, &s->bucket_attrs);
4970   if (ret < 0) {
4971       // TODO-FUTURE: if the bucket does not exist, maybe expose it here?
4972       return -ERR_NO_SUCH_BUCKET;
4973   }
4974   if (!s->bucket_info.has_website) {
4975       // TODO-FUTURE: if the bucket has no WebsiteConfig, expose it here
4976       return -ERR_NO_SUCH_WEBSITE_CONFIGURATION;
4977   }
4978
4979   rgw_obj_key new_obj;
4980   bool get_res = s->bucket_info.website_conf.get_effective_key(s->object.name, &new_obj.name, web_dir());
4981   if (!get_res) {
4982     s->err.message = "The IndexDocument Suffix is not configurated or not well formed!";
4983     ldpp_dout(s, 5) << s->err.message << dendl;
4984     return -EINVAL;
4985   }
4986
4987   ldpp_dout(s, 10) << "retarget get_effective_key " << s->object << " -> "
4988                     << new_obj << dendl;
4989
4990   RGWBWRoutingRule rrule;
4991   bool should_redirect =
4992     s->bucket_info.website_conf.should_redirect(new_obj.name, 0, &rrule);
4993
4994   if (should_redirect) {
4995     const string& hostname = s->info.env->get("HTTP_HOST", "");
4996     const string& protocol =
4997       (s->info.env->get("SERVER_PORT_SECURE") ? "https" : "http");
4998     int redirect_code = 0;
4999     rrule.apply_rule(protocol, hostname, s->object.name, &s->redirect,
5000                     &redirect_code);
5001     // APply a custom HTTP response code
5002     if (redirect_code > 0)
5003       s->err.http_ret = redirect_code; // Apply a custom HTTP response code
5004     ldpp_dout(s, 10) << "retarget redirect code=" << redirect_code
5005                       << " proto+host:" << protocol << "://" << hostname
5006                       << " -> " << s->redirect << dendl;
5007     return -ERR_WEBSITE_REDIRECT;
5008   }
5009
5010   /*
5011    * FIXME: if s->object != new_obj, drop op and create a new op to handle
5012    * operation. Or remove this comment if it's not applicable anymore
5013    */
5014
5015   s->object = new_obj;
5016
5017   return 0;
5018 }
5019
5020 RGWOp* RGWHandler_REST_S3Website::op_get()
5021 {
5022   return get_obj_op(true);
5023 }
5024
5025 RGWOp* RGWHandler_REST_S3Website::op_head()
5026 {
5027   return get_obj_op(false);
5028 }
5029
5030 int RGWHandler_REST_S3Website::serve_errordoc(int http_ret, const string& errordoc_key) {
5031   int ret = 0;
5032   s->formatter->reset(); /* Try to throw it all away */
5033
5034   std::shared_ptr<RGWGetObj_ObjStore_S3Website> getop( static_cast<RGWGetObj_ObjStore_S3Website*>(op_get()));
5035   if (getop.get() == NULL) {
5036     return -1; // Trigger double error handler
5037   }
5038   getop->init(store, s, this);
5039   getop->range_str = NULL;
5040   getop->if_mod = NULL;
5041   getop->if_unmod = NULL;
5042   getop->if_match = NULL;
5043   getop->if_nomatch = NULL;
5044   s->object = errordoc_key;
5045
5046   ret = init_permissions(getop.get());
5047   if (ret < 0) {
5048     ldpp_dout(s, 20) << "serve_errordoc failed, init_permissions ret=" << ret << dendl;
5049     return -1; // Trigger double error handler
5050   }
5051
5052   ret = read_permissions(getop.get());
5053   if (ret < 0) {
5054     ldpp_dout(s, 20) << "serve_errordoc failed, read_permissions ret=" << ret << dendl;
5055     return -1; // Trigger double error handler
5056   }
5057
5058   if (http_ret) {
5059      getop->set_custom_http_response(http_ret);
5060   }
5061
5062   ret = getop->init_processing();
5063   if (ret < 0) {
5064     ldpp_dout(s, 20) << "serve_errordoc failed, init_processing ret=" << ret << dendl;
5065     return -1; // Trigger double error handler
5066   }
5067
5068   ret = getop->verify_op_mask();
5069   if (ret < 0) {
5070     ldpp_dout(s, 20) << "serve_errordoc failed, verify_op_mask ret=" << ret << dendl;
5071     return -1; // Trigger double error handler
5072   }
5073
5074   ret = getop->verify_permission();
5075   if (ret < 0) {
5076     ldpp_dout(s, 20) << "serve_errordoc failed, verify_permission ret=" << ret << dendl;
5077     return -1; // Trigger double error handler
5078   }
5079
5080   ret = getop->verify_params();
5081   if (ret < 0) {
5082     ldpp_dout(s, 20) << "serve_errordoc failed, verify_params ret=" << ret << dendl;
5083     return -1; // Trigger double error handler
5084   }
5085
5086   // No going back now
5087   getop->pre_exec();
5088   /*
5089    * FIXME Missing headers:
5090    * With a working errordoc, the s3 error fields are rendered as HTTP headers,
5091    *   x-amz-error-code: NoSuchKey
5092    *   x-amz-error-message: The specified key does not exist.
5093    *   x-amz-error-detail-Key: foo
5094    */
5095   getop->execute();
5096   getop->complete();
5097   return 0;
5098
5099 }
5100
5101 int RGWHandler_REST_S3Website::error_handler(int err_no,
5102                                             string* error_content) {
5103   int new_err_no = -1;
5104   rgw_http_errors::const_iterator r = rgw_http_s3_errors.find(err_no > 0 ? err_no : -err_no);
5105   int http_error_code = -1;
5106
5107   if (r != rgw_http_s3_errors.end()) {
5108     http_error_code = r->second.first;
5109   }
5110   ldpp_dout(s, 10) << "RGWHandler_REST_S3Website::error_handler err_no=" << err_no << " http_ret=" << http_error_code << dendl;
5111
5112   RGWBWRoutingRule rrule;
5113   bool should_redirect =
5114     s->bucket_info.website_conf.should_redirect(original_object_name,
5115                                                 http_error_code, &rrule);
5116
5117   if (should_redirect) {
5118     const string& hostname = s->info.env->get("HTTP_HOST", "");
5119     const string& protocol =
5120       (s->info.env->get("SERVER_PORT_SECURE") ? "https" : "http");
5121     int redirect_code = 0;
5122     rrule.apply_rule(protocol, hostname, original_object_name,
5123                      &s->redirect, &redirect_code);
5124     // Apply a custom HTTP response code
5125     if (redirect_code > 0)
5126       s->err.http_ret = redirect_code; // Apply a custom HTTP response code
5127     ldpp_dout(s, 10) << "error handler redirect code=" << redirect_code
5128                       << " proto+host:" << protocol << "://" << hostname
5129                       << " -> " << s->redirect << dendl;
5130     return -ERR_WEBSITE_REDIRECT;
5131   } else if (err_no == -ERR_WEBSITE_REDIRECT) {
5132     // Do nothing here, this redirect will be handled in abort_early's ERR_WEBSITE_REDIRECT block
5133     // Do NOT fire the ErrorDoc handler
5134   } else if (!s->bucket_info.website_conf.error_doc.empty()) {
5135     /* This serves an entire page!
5136        On success, it will return zero, and no further content should be sent to the socket
5137        On failure, we need the double-error handler
5138      */
5139     new_err_no = RGWHandler_REST_S3Website::serve_errordoc(http_error_code, s->bucket_info.website_conf.error_doc);
5140     if (new_err_no != -1) {
5141       err_no = new_err_no;
5142     }
5143   } else {
5144     ldpp_dout(s, 20) << "No special error handling today!" << dendl;
5145   }
5146
5147   return err_no;
5148 }
5149
5150 RGWOp* RGWHandler_REST_Obj_S3Website::get_obj_op(bool get_data)
5151 {
5152   /** If we are in website mode, then it is explicitly impossible to run GET or
5153    * HEAD on the actual directory. We must convert the request to run on the
5154    * suffix object instead!
5155    */
5156   RGWGetObj_ObjStore_S3Website* op = new RGWGetObj_ObjStore_S3Website;
5157   op->set_get_data(get_data);
5158   return op;
5159 }
5160
5161 RGWOp* RGWHandler_REST_Bucket_S3Website::get_obj_op(bool get_data)
5162 {
5163   /** If we are in website mode, then it is explicitly impossible to run GET or
5164    * HEAD on the actual directory. We must convert the request to run on the
5165    * suffix object instead!
5166    */
5167   RGWGetObj_ObjStore_S3Website* op = new RGWGetObj_ObjStore_S3Website;
5168   op->set_get_data(get_data);
5169   return op;
5170 }
5171
5172 RGWOp* RGWHandler_REST_Service_S3Website::get_obj_op(bool get_data)
5173 {
5174   /** If we are in website mode, then it is explicitly impossible to run GET or
5175    * HEAD on the actual directory. We must convert the request to run on the
5176    * suffix object instead!
5177    */
5178   RGWGetObj_ObjStore_S3Website* op = new RGWGetObj_ObjStore_S3Website;
5179   op->set_get_data(get_data);
5180   return op;
5181 }
5182
5183
5184 namespace rgw::auth::s3 {
5185
5186 static rgw::auth::Completer::cmplptr_t
5187 null_completer_factory(const boost::optional<std::string>& secret_key)
5188 {
5189   return nullptr;
5190 }
5191
5192
5193 AWSEngine::VersionAbstractor::auth_data_t
5194 AWSGeneralAbstractor::get_auth_data(const req_state* const s) const
5195 {
5196   AwsVersion version;
5197   AwsRoute route;
5198   std::tie(version, route) = discover_aws_flavour(s->info);
5199
5200   if (version == AwsVersion::V2) {
5201     return get_auth_data_v2(s);
5202   } else if (version == AwsVersion::V4) {
5203     return get_auth_data_v4(s, route == AwsRoute::QUERY_STRING);
5204   } else {
5205     /* FIXME(rzarzynski): handle anon user. */
5206     throw -EINVAL;
5207   }
5208 }
5209
5210 boost::optional<std::string>
5211 AWSGeneralAbstractor::get_v4_canonical_headers(
5212   const req_info& info,
5213   const boost::string_view& signedheaders,
5214   const bool using_qs) const
5215 {
5216   return rgw::auth::s3::get_v4_canonical_headers(info, signedheaders,
5217                                                  using_qs, false);
5218 }
5219
5220 AWSEngine::VersionAbstractor::auth_data_t
5221 AWSGeneralAbstractor::get_auth_data_v4(const req_state* const s,
5222                                        const bool using_qs) const
5223 {
5224   boost::string_view access_key_id;
5225   boost::string_view signed_hdrs;
5226
5227   boost::string_view date;
5228   boost::string_view credential_scope;
5229   boost::string_view client_signature;
5230   boost::string_view session_token;
5231
5232   int ret = rgw::auth::s3::parse_v4_credentials(s->info,
5233                                                 access_key_id,
5234                                                 credential_scope,
5235                                                 signed_hdrs,
5236                                                 client_signature,
5237                                                 date,
5238                                                 session_token,
5239                                                 using_qs);
5240   if (ret < 0) {
5241     throw ret;
5242   }
5243
5244   /* craft canonical headers */
5245   boost::optional<std::string> canonical_headers = \
5246     get_v4_canonical_headers(s->info, signed_hdrs, using_qs);
5247   if (canonical_headers) {
5248     using sanitize = rgw::crypt_sanitize::log_content;
5249     ldpp_dout(s, 10) << "canonical headers format = "
5250                       << sanitize{*canonical_headers} << dendl;
5251   } else {
5252     throw -EPERM;
5253   }
5254
5255   bool is_non_s3_op = false;
5256   if (s->op_type == RGW_STS_GET_SESSION_TOKEN ||
5257       s->op_type == RGW_STS_ASSUME_ROLE ||
5258       s->op_type == RGW_STS_ASSUME_ROLE_WEB_IDENTITY ||
5259       s->op_type == RGW_OP_CREATE_ROLE ||
5260       s->op_type == RGW_OP_DELETE_ROLE ||
5261       s->op_type == RGW_OP_GET_ROLE ||
5262       s->op_type == RGW_OP_MODIFY_ROLE ||
5263       s->op_type == RGW_OP_LIST_ROLES ||
5264       s->op_type == RGW_OP_PUT_ROLE_POLICY ||
5265       s->op_type == RGW_OP_GET_ROLE_POLICY ||
5266       s->op_type == RGW_OP_LIST_ROLE_POLICIES ||
5267       s->op_type == RGW_OP_DELETE_ROLE_POLICY ||
5268       s->op_type == RGW_OP_PUT_USER_POLICY ||
5269       s->op_type == RGW_OP_GET_USER_POLICY ||
5270       s->op_type == RGW_OP_LIST_USER_POLICIES ||
5271       s->op_type == RGW_OP_DELETE_USER_POLICY ||
5272       s->op_type == RGW_OP_CREATE_OIDC_PROVIDER ||
5273       s->op_type == RGW_OP_DELETE_OIDC_PROVIDER ||
5274       s->op_type == RGW_OP_GET_OIDC_PROVIDER ||
5275       s->op_type == RGW_OP_LIST_OIDC_PROVIDERS) {
5276     is_non_s3_op = true;
5277   }
5278
5279   const char* exp_payload_hash = nullptr;
5280   string payload_hash;
5281   if (is_non_s3_op) {
5282     //For non s3 ops, we need to calculate the payload hash
5283     payload_hash = s->info.args.get("PayloadHash");
5284     exp_payload_hash = payload_hash.c_str();
5285   } else {
5286     /* Get the expected hash. */
5287     exp_payload_hash = rgw::auth::s3::get_v4_exp_payload_hash(s->info);
5288   }
5289
5290   /* Craft canonical URI. Using std::move later so let it be non-const. */
5291   auto canonical_uri = rgw::auth::s3::get_v4_canonical_uri(s->info);
5292
5293   /* Craft canonical query string. std::moving later so non-const here. */
5294   auto canonical_qs = rgw::auth::s3::get_v4_canonical_qs(s->info, using_qs);
5295
5296   /* Craft canonical request. */
5297   auto canonical_req_hash = \
5298     rgw::auth::s3::get_v4_canon_req_hash(s->cct,
5299                                          s->info.method,
5300                                          std::move(canonical_uri),
5301                                          std::move(canonical_qs),
5302                                          std::move(*canonical_headers),
5303                                          signed_hdrs,
5304                                          exp_payload_hash);
5305
5306   auto string_to_sign = \
5307     rgw::auth::s3::get_v4_string_to_sign(s->cct,
5308                                          AWS4_HMAC_SHA256_STR,
5309                                          date,
5310                                          credential_scope,
5311                                          std::move(canonical_req_hash));
5312
5313   const auto sig_factory = std::bind(rgw::auth::s3::get_v4_signature,
5314                                      credential_scope,
5315                                      std::placeholders::_1,
5316                                      std::placeholders::_2,
5317                                      std::placeholders::_3);
5318
5319   /* Requests authenticated with the Query Parameters are treated as unsigned.
5320    * From "Authenticating Requests: Using Query Parameters (AWS Signature
5321    * Version 4)":
5322    *
5323    *   You don't include a payload hash in the Canonical Request, because
5324    *   when you create a presigned URL, you don't know the payload content
5325    *   because the URL is used to upload an arbitrary payload. Instead, you
5326    *   use a constant string UNSIGNED-PAYLOAD.
5327    *
5328    * This means we have absolutely no business in spawning completer. Both
5329    * aws4_auth_needs_complete and aws4_auth_streaming_mode are set to false
5330    * by default. We don't need to change that. */
5331   if (is_v4_payload_unsigned(exp_payload_hash) || is_v4_payload_empty(s) || is_non_s3_op) {
5332     return {
5333       access_key_id,
5334       client_signature,
5335       session_token,
5336       std::move(string_to_sign),
5337       sig_factory,
5338       null_completer_factory
5339     };
5340   } else {
5341     /* We're going to handle a signed payload. Be aware that even empty HTTP
5342      * body (no payload) requires verification:
5343      *
5344      *   The x-amz-content-sha256 header is required for all AWS Signature
5345      *   Version 4 requests. It provides a hash of the request payload. If
5346      *   there is no payload, you must provide the hash of an empty string. */
5347     if (!is_v4_payload_streamed(exp_payload_hash)) {
5348       ldpp_dout(s, 10) << "delaying v4 auth" << dendl;
5349
5350       /* payload in a single chunk */
5351       switch (s->op_type)
5352       {
5353         case RGW_OP_CREATE_BUCKET:
5354         case RGW_OP_PUT_OBJ:
5355         case RGW_OP_PUT_ACLS:
5356         case RGW_OP_PUT_CORS:
5357         case RGW_OP_INIT_MULTIPART: // in case that Init Multipart uses CHUNK encoding
5358         case RGW_OP_COMPLETE_MULTIPART:
5359         case RGW_OP_SET_BUCKET_VERSIONING:
5360         case RGW_OP_DELETE_MULTI_OBJ:
5361         case RGW_OP_ADMIN_SET_METADATA:
5362         case RGW_OP_SET_BUCKET_WEBSITE:
5363         case RGW_OP_PUT_BUCKET_POLICY:
5364         case RGW_OP_PUT_OBJ_TAGGING:
5365         case RGW_OP_PUT_BUCKET_TAGGING:
5366         case RGW_OP_PUT_BUCKET_REPLICATION:
5367         case RGW_OP_PUT_LC:
5368         case RGW_OP_SET_REQUEST_PAYMENT:
5369         case RGW_OP_PUBSUB_NOTIF_CREATE:
5370         case RGW_OP_PUT_BUCKET_OBJ_LOCK:
5371         case RGW_OP_PUT_OBJ_RETENTION:
5372         case RGW_OP_PUT_OBJ_LEGAL_HOLD:
5373         case RGW_STS_GET_SESSION_TOKEN:
5374         case RGW_STS_ASSUME_ROLE:
5375         case RGW_OP_PUT_BUCKET_PUBLIC_ACCESS_BLOCK:
5376         case RGW_OP_GET_BUCKET_PUBLIC_ACCESS_BLOCK:
5377         case RGW_OP_DELETE_BUCKET_PUBLIC_ACCESS_BLOCK:
5378           break;
5379         default:
5380           dout(10) << "ERROR: AWS4 completion for this operation NOT IMPLEMENTED" << dendl;
5381           throw -ERR_NOT_IMPLEMENTED;
5382       }
5383
5384       const auto cmpl_factory = std::bind(AWSv4ComplSingle::create,
5385                                           s,
5386                                           std::placeholders::_1);
5387       return {
5388         access_key_id,
5389         client_signature,
5390         session_token,
5391         std::move(string_to_sign),
5392         sig_factory,
5393         cmpl_factory
5394       };
5395     } else {
5396       /* IMHO "streamed" doesn't fit too good here. I would prefer to call
5397        * it "chunked" but let's be coherent with Amazon's terminology. */
5398
5399       dout(10) << "body content detected in multiple chunks" << dendl;
5400
5401       /* payload in multiple chunks */
5402
5403       switch(s->op_type)
5404       {
5405         case RGW_OP_PUT_OBJ:
5406           break;
5407         default:
5408           dout(10) << "ERROR: AWS4 completion for this operation NOT IMPLEMENTED (streaming mode)" << dendl;
5409           throw -ERR_NOT_IMPLEMENTED;
5410       }
5411
5412       dout(10) << "aws4 seed signature ok... delaying v4 auth" << dendl;
5413
5414       /* In the case of streamed payload client sets the x-amz-content-sha256
5415        * to "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" but uses "UNSIGNED-PAYLOAD"
5416        * when constructing the Canonical Request. */
5417
5418       /* In the case of single-chunk upload client set the header's value is
5419        * coherent with the one used for Canonical Request crafting. */
5420
5421       /* In the case of query string-based authentication there should be no
5422        * x-amz-content-sha256 header and the value "UNSIGNED-PAYLOAD" is used
5423        * for CanonReq. */
5424       const auto cmpl_factory = std::bind(AWSv4ComplMulti::create,
5425                                           s,
5426                                           date,
5427                                           credential_scope,
5428                                           client_signature,
5429                                           std::placeholders::_1);
5430       return {
5431         access_key_id,
5432         client_signature,
5433         session_token,
5434         std::move(string_to_sign),
5435         sig_factory,
5436         cmpl_factory
5437       };
5438     }
5439   }
5440 }
5441
5442
5443 boost::optional<std::string>
5444 AWSGeneralBoto2Abstractor::get_v4_canonical_headers(
5445   const req_info& info,
5446   const boost::string_view& signedheaders,
5447   const bool using_qs) const
5448 {
5449   return rgw::auth::s3::get_v4_canonical_headers(info, signedheaders,
5450                                                  using_qs, true);
5451 }
5452
5453
5454 AWSEngine::VersionAbstractor::auth_data_t
5455 AWSGeneralAbstractor::get_auth_data_v2(const req_state* const s) const
5456 {
5457   boost::string_view access_key_id;
5458   boost::string_view signature;
5459   boost::string_view session_token;
5460   bool qsr = false;
5461
5462   const char* http_auth = s->info.env->get("HTTP_AUTHORIZATION");
5463   if (! http_auth || http_auth[0] == '\0') {
5464     /* Credentials are provided in query string. We also need to verify
5465      * the "Expires" parameter now. */
5466     access_key_id = s->info.args.get("AWSAccessKeyId");
5467     signature = s->info.args.get("Signature");
5468     qsr = true;
5469
5470     boost::string_view expires = s->info.args.get("Expires");
5471     if (expires.empty()) {
5472       throw -EPERM;
5473     }
5474
5475     /* It looks we have the guarantee that expires is a null-terminated,
5476      * and thus string_view::data() can be safely used. */
5477     const time_t exp = atoll(expires.data());
5478     time_t now;
5479     time(&now);
5480
5481     if (now >= exp) {
5482       throw -EPERM;
5483     }
5484     if (s->info.args.exists("X-Amz-Security-Token")) {
5485       session_token = s->info.args.get("X-Amz-Security-Token");
5486       if (session_token.size() == 0) {
5487         throw -EPERM;
5488       }
5489     }
5490
5491   } else {
5492     /* The "Authorization" HTTP header is being used. */
5493     const boost::string_view auth_str(http_auth + strlen("AWS "));
5494     const size_t pos = auth_str.rfind(':');
5495     if (pos != boost::string_view::npos) {
5496       access_key_id = auth_str.substr(0, pos);
5497       signature = auth_str.substr(pos + 1);
5498     }
5499
5500     if (s->info.env->exists("HTTP_X_AMZ_SECURITY_TOKEN")) {
5501       session_token = s->info.env->get("HTTP_X_AMZ_SECURITY_TOKEN");
5502       if (session_token.size() == 0) {
5503         throw -EPERM;
5504       }
5505     }
5506   }
5507
5508   /* Let's canonize the HTTP headers that are covered by the AWS auth v2. */
5509   std::string string_to_sign;
5510   utime_t header_time;
5511   if (! rgw_create_s3_canonical_header(s->info, &header_time, string_to_sign,
5512         qsr)) {
5513     ldpp_dout(s, 10) << "failed to create the canonized auth header\n"
5514                    << rgw::crypt_sanitize::auth{s,string_to_sign} << dendl;
5515     throw -EPERM;
5516   }
5517
5518   ldpp_dout(s, 10) << "string_to_sign:\n"
5519                  << rgw::crypt_sanitize::auth{s,string_to_sign} << dendl;
5520
5521   if (!qsr && !is_time_skew_ok(header_time)) {
5522     throw -ERR_REQUEST_TIME_SKEWED;
5523   }
5524
5525   return {
5526     std::move(access_key_id),
5527     std::move(signature),
5528     std::move(session_token),
5529     std::move(string_to_sign),
5530     rgw::auth::s3::get_v2_signature,
5531     null_completer_factory
5532   };
5533 }
5534
5535
5536 AWSEngine::VersionAbstractor::auth_data_t
5537 AWSBrowserUploadAbstractor::get_auth_data_v2(const req_state* const s) const
5538 {
5539   return {
5540     s->auth.s3_postobj_creds.access_key,
5541     s->auth.s3_postobj_creds.signature,
5542     s->auth.s3_postobj_creds.x_amz_security_token,
5543     s->auth.s3_postobj_creds.encoded_policy.to_str(),
5544     rgw::auth::s3::get_v2_signature,
5545     null_completer_factory
5546   };
5547 }
5548
5549 AWSEngine::VersionAbstractor::auth_data_t
5550 AWSBrowserUploadAbstractor::get_auth_data_v4(const req_state* const s) const
5551 {
5552   const boost::string_view credential = s->auth.s3_postobj_creds.x_amz_credential;
5553
5554   /* grab access key id */
5555   const size_t pos = credential.find("/");
5556   const boost::string_view access_key_id = credential.substr(0, pos);
5557   dout(10) << "access key id = " << access_key_id << dendl;
5558
5559   /* grab credential scope */
5560   const boost::string_view credential_scope = credential.substr(pos + 1);
5561   dout(10) << "credential scope = " << credential_scope << dendl;
5562
5563   const auto sig_factory = std::bind(rgw::auth::s3::get_v4_signature,
5564                                      credential_scope,
5565                                      std::placeholders::_1,
5566                                      std::placeholders::_2,
5567                                      std::placeholders::_3);
5568
5569   return {
5570     access_key_id,
5571     s->auth.s3_postobj_creds.signature,
5572     s->auth.s3_postobj_creds.x_amz_security_token,
5573     s->auth.s3_postobj_creds.encoded_policy.to_str(),
5574     sig_factory,
5575     null_completer_factory
5576   };
5577 }
5578
5579 AWSEngine::VersionAbstractor::auth_data_t
5580 AWSBrowserUploadAbstractor::get_auth_data(const req_state* const s) const
5581 {
5582   if (s->auth.s3_postobj_creds.x_amz_algorithm == AWS4_HMAC_SHA256_STR) {
5583     ldpp_dout(s, 0) << "Signature verification algorithm AWS v4"
5584                      << " (AWS4-HMAC-SHA256)" << dendl;
5585     return get_auth_data_v4(s);
5586   } else {
5587     ldpp_dout(s, 0) << "Signature verification algorithm AWS v2" << dendl;
5588     return get_auth_data_v2(s);
5589   }
5590 }
5591
5592 AWSEngine::result_t
5593 AWSEngine::authenticate(const DoutPrefixProvider* dpp, const req_state* const s) const
5594 {
5595   /* Small reminder: an ver_abstractor is allowed to throw! */
5596   const auto auth_data = ver_abstractor.get_auth_data(s);
5597
5598   if (auth_data.access_key_id.empty() || auth_data.client_signature.empty()) {
5599     return result_t::deny(-EINVAL);
5600   } else {
5601     return authenticate(dpp,
5602                         auth_data.access_key_id,
5603                         auth_data.client_signature,
5604             auth_data.session_token,
5605                         auth_data.string_to_sign,
5606                         auth_data.signature_factory,
5607                         auth_data.completer_factory,
5608                         s);
5609   }
5610 }
5611
5612 } // namespace rgw::auth::s3
5613
5614 rgw::LDAPHelper* rgw::auth::s3::LDAPEngine::ldh = nullptr;
5615 std::mutex rgw::auth::s3::LDAPEngine::mtx;
5616
5617 void rgw::auth::s3::LDAPEngine::init(CephContext* const cct)
5618 {
5619   if (! cct->_conf->rgw_s3_auth_use_ldap ||
5620       cct->_conf->rgw_ldap_uri.empty()) {
5621     return;
5622   }
5623
5624   if (! ldh) {
5625     std::lock_guard<std::mutex> lck(mtx);
5626     if (! ldh) {
5627       const string& ldap_uri = cct->_conf->rgw_ldap_uri;
5628       const string& ldap_binddn = cct->_conf->rgw_ldap_binddn;
5629       const string& ldap_searchdn = cct->_conf->rgw_ldap_searchdn;
5630       const string& ldap_searchfilter = cct->_conf->rgw_ldap_searchfilter;
5631       const string& ldap_dnattr = cct->_conf->rgw_ldap_dnattr;
5632       std::string ldap_bindpw = parse_rgw_ldap_bindpw(cct);
5633
5634       ldh = new rgw::LDAPHelper(ldap_uri, ldap_binddn, ldap_bindpw,
5635                                 ldap_searchdn, ldap_searchfilter, ldap_dnattr);
5636
5637       ldh->init();
5638       ldh->bind();
5639     }
5640   }
5641 }
5642
5643 bool rgw::auth::s3::LDAPEngine::valid() {
5644   std::lock_guard<std::mutex> lck(mtx);
5645   return (!!ldh);
5646 }
5647
5648 rgw::auth::RemoteApplier::acl_strategy_t
5649 rgw::auth::s3::LDAPEngine::get_acl_strategy() const
5650 {
5651   //This is based on the assumption that the default acl strategy in
5652   // get_perms_from_aclspec, will take care. Extra acl spec is not required.
5653   return nullptr;
5654 }
5655
5656 rgw::auth::RemoteApplier::AuthInfo
5657 rgw::auth::s3::LDAPEngine::get_creds_info(const rgw::RGWToken& token) const noexcept
5658 {
5659   /* The short form of "using" can't be used here -- we're aliasing a class'
5660    * member. */
5661   using acct_privilege_t = \
5662     rgw::auth::RemoteApplier::AuthInfo::acct_privilege_t;
5663
5664   return rgw::auth::RemoteApplier::AuthInfo {
5665     rgw_user(token.id),
5666     token.id,
5667     RGW_PERM_FULL_CONTROL,
5668     acct_privilege_t::IS_PLAIN_ACCT,
5669     TYPE_LDAP
5670   };
5671 }
5672
5673 rgw::auth::Engine::result_t
5674 rgw::auth::s3::LDAPEngine::authenticate(
5675   const DoutPrefixProvider* dpp,
5676   const boost::string_view& access_key_id,
5677   const boost::string_view& signature,
5678   const boost::string_view& session_token,
5679   const string_to_sign_t& string_to_sign,
5680   const signature_factory_t&,
5681   const completer_factory_t& completer_factory,
5682   const req_state* const s) const
5683 {
5684   /* boost filters and/or string_ref may throw on invalid input */
5685   rgw::RGWToken base64_token;
5686   try {
5687     base64_token = rgw::from_base64(access_key_id);
5688   } catch (...) {
5689     base64_token = std::string("");
5690   }
5691
5692   if (! base64_token.valid()) {
5693     return result_t::deny();
5694   }
5695
5696   //TODO: Uncomment, when we have a migration plan in place.
5697   //Check if a user of type other than 'ldap' is already present, if yes, then
5698   //return error.
5699   /*RGWUserInfo user_info;
5700   user_info.user_id = base64_token.id;
5701   if (rgw_get_user_info_by_uid(store, user_info.user_id, user_info) >= 0) {
5702     if (user_info.type != TYPE_LDAP) {
5703       ldpp_dout(dpp, 10) << "ERROR: User id of type: " << user_info.type << " is already present" << dendl;
5704       return nullptr;
5705     }
5706   }*/
5707
5708   if (ldh->auth(base64_token.id, base64_token.key) != 0) {
5709     return result_t::deny(-ERR_INVALID_ACCESS_KEY);
5710   }
5711
5712   auto apl = apl_factory->create_apl_remote(cct, s, get_acl_strategy(),
5713                                             get_creds_info(base64_token));
5714   return result_t::grant(std::move(apl), completer_factory(boost::none));
5715 } /* rgw::auth::s3::LDAPEngine::authenticate */
5716
5717 void rgw::auth::s3::LDAPEngine::shutdown() {
5718   if (ldh) {
5719     delete ldh;
5720     ldh = nullptr;
5721   }
5722 }
5723
5724 /* LocalEngine */
5725 rgw::auth::Engine::result_t
5726 rgw::auth::s3::LocalEngine::authenticate(
5727   const DoutPrefixProvider* dpp,
5728   const boost::string_view& _access_key_id,
5729   const boost::string_view& signature,
5730   const boost::string_view& session_token,
5731   const string_to_sign_t& string_to_sign,
5732   const signature_factory_t& signature_factory,
5733   const completer_factory_t& completer_factory,
5734   const req_state* const s) const
5735 {
5736   /* get the user info */
5737   RGWUserInfo user_info;
5738   /* TODO(rzarzynski): we need to have string-view taking variant. */
5739   const std::string access_key_id = _access_key_id.to_string();
5740   if (rgw_get_user_info_by_access_key(ctl->user, access_key_id, user_info) < 0) {
5741       ldpp_dout(dpp, 5) << "error reading user info, uid=" << access_key_id
5742               << " can't authenticate" << dendl;
5743       return result_t::deny(-ERR_INVALID_ACCESS_KEY);
5744   }
5745   //TODO: Uncomment, when we have a migration plan in place.
5746   /*else {
5747     if (s->user->type != TYPE_RGW) {
5748       ldpp_dout(dpp, 10) << "ERROR: User id of type: " << s->user->type
5749                      << " is present" << dendl;
5750       throw -EPERM;
5751     }
5752   }*/
5753
5754   const auto iter = user_info.access_keys.find(access_key_id);
5755   if (iter == std::end(user_info.access_keys)) {
5756     ldpp_dout(dpp, 0) << "ERROR: access key not encoded in user info" << dendl;
5757     return result_t::deny(-EPERM);
5758   }
5759   const RGWAccessKey& k = iter->second;
5760
5761   const VersionAbstractor::server_signature_t server_signature = \
5762     signature_factory(cct, k.key, string_to_sign);
5763   auto compare = signature.compare(server_signature);
5764
5765   ldpp_dout(dpp, 15) << "string_to_sign="
5766                  << rgw::crypt_sanitize::log_content{string_to_sign}
5767                  << dendl;
5768   ldpp_dout(dpp, 15) << "server signature=" << server_signature << dendl;
5769   ldpp_dout(dpp, 15) << "client signature=" << signature << dendl;
5770   ldpp_dout(dpp, 15) << "compare=" << compare << dendl;
5771
5772   if (compare != 0) {
5773     return result_t::deny(-ERR_SIGNATURE_NO_MATCH);
5774   }
5775
5776   auto apl = apl_factory->create_apl_local(cct, s, user_info, k.subuser, boost::none);
5777   return result_t::grant(std::move(apl), completer_factory(k.key));
5778 }
5779
5780 rgw::auth::RemoteApplier::AuthInfo
5781 rgw::auth::s3::STSEngine::get_creds_info(const STS::SessionToken& token) const noexcept
5782 {
5783   using acct_privilege_t = \
5784     rgw::auth::RemoteApplier::AuthInfo::acct_privilege_t;
5785
5786   return rgw::auth::RemoteApplier::AuthInfo {
5787     token.user,
5788     token.acct_name,
5789     token.perm_mask,
5790     (token.is_admin) ? acct_privilege_t::IS_ADMIN_ACCT: acct_privilege_t::IS_PLAIN_ACCT,
5791     token.acct_type
5792   };
5793 }
5794
5795 int
5796 rgw::auth::s3::STSEngine::get_session_token(const DoutPrefixProvider* dpp, const boost::string_view& session_token,
5797                                             STS::SessionToken& token) const
5798 {
5799   string decodedSessionToken;
5800   try {
5801     decodedSessionToken = rgw::from_base64(session_token);
5802   } catch (...) {
5803     ldpp_dout(dpp, 0) << "ERROR: Invalid session token, not base64 encoded." << dendl;
5804     return -EINVAL;
5805   }
5806
5807   auto* cryptohandler = cct->get_crypto_handler(CEPH_CRYPTO_AES);
5808   if (! cryptohandler) {
5809     return -EINVAL;
5810   }
5811   string secret_s = cct->_conf->rgw_sts_key;
5812   buffer::ptr secret(secret_s.c_str(), secret_s.length());
5813   int ret = 0;
5814   if (ret = cryptohandler->validate_secret(secret); ret < 0) {
5815     ldpp_dout(dpp, 0) << "ERROR: Invalid secret key" << dendl;
5816     return -EINVAL;
5817   }
5818   string error;
5819   auto* keyhandler = cryptohandler->get_key_handler(secret, error);
5820   if (! keyhandler) {
5821     return -EINVAL;
5822   }
5823   error.clear();
5824
5825   string decrypted_str;
5826   buffer::list en_input, dec_output;
5827   en_input = buffer::list::static_from_string(decodedSessionToken);
5828
5829   ret = keyhandler->decrypt(en_input, dec_output, &error);
5830   if (ret < 0) {
5831     ldpp_dout(dpp, 0) << "ERROR: Decryption failed: " << error << dendl;
5832     return -EPERM;
5833   } else {
5834     try {
5835       dec_output.append('\0');
5836       auto iter = dec_output.cbegin();
5837       decode(token, iter);
5838     } catch (const buffer::error& e) {
5839       ldout(cct, 0) << "ERROR: decode SessionToken failed: " << error << dendl;
5840       return -EINVAL;
5841     }
5842   }
5843   return 0;
5844 }
5845
5846 rgw::auth::Engine::result_t
5847 rgw::auth::s3::STSEngine::authenticate(
5848   const DoutPrefixProvider* dpp,
5849   const boost::string_view& _access_key_id,
5850   const boost::string_view& signature,
5851   const boost::string_view& session_token,
5852   const string_to_sign_t& string_to_sign,
5853   const signature_factory_t& signature_factory,
5854   const completer_factory_t& completer_factory,
5855   const req_state* const s) const
5856 {
5857   if (! s->info.args.exists("X-Amz-Security-Token") &&
5858       ! s->info.env->exists("HTTP_X_AMZ_SECURITY_TOKEN") &&
5859       s->auth.s3_postobj_creds.x_amz_security_token.empty()) {
5860     return result_t::deny();
5861   }
5862
5863   STS::SessionToken token;
5864   if (int ret = get_session_token(dpp, session_token, token); ret < 0) {
5865     return result_t::reject(ret);
5866   }
5867   //Authentication
5868   //Check if access key is not the same passed in by client
5869   if (token.access_key_id != _access_key_id) {
5870     ldpp_dout(dpp, 0) << "Invalid access key" << dendl;
5871     return result_t::reject(-EPERM);
5872   }
5873   //Check if the token has expired
5874   if (! token.expiration.empty()) {
5875     std::string expiration = token.expiration;
5876     if (! expiration.empty()) {
5877       boost::optional<real_clock::time_point> exp = ceph::from_iso_8601(expiration, false);
5878       if (exp) {
5879         real_clock::time_point now = real_clock::now();
5880         if (now >= *exp) {
5881           ldpp_dout(dpp, 0) << "ERROR: Token expired" << dendl;
5882           return result_t::reject(-EPERM);
5883         }
5884       } else {
5885         ldpp_dout(dpp, 0) << "ERROR: Invalid expiration: " << expiration << dendl;
5886         return result_t::reject(-EPERM);
5887       }
5888     }
5889   }
5890   //Check for signature mismatch
5891   const VersionAbstractor::server_signature_t server_signature = \
5892     signature_factory(cct, token.secret_access_key, string_to_sign);
5893   auto compare = signature.compare(server_signature);
5894
5895   ldpp_dout(dpp, 15) << "string_to_sign="
5896                  << rgw::crypt_sanitize::log_content{string_to_sign}
5897                  << dendl;
5898   ldpp_dout(dpp, 15) << "server signature=" << server_signature << dendl;
5899   ldpp_dout(dpp, 15) << "client signature=" << signature << dendl;
5900   ldpp_dout(dpp, 15) << "compare=" << compare << dendl;
5901
5902   if (compare != 0) {
5903     return result_t::reject(-ERR_SIGNATURE_NO_MATCH);
5904   }
5905
5906   // Get all the authorization info
5907   RGWUserInfo user_info;
5908   rgw_user user_id;
5909   string role_id;
5910   rgw::auth::RoleApplier::Role r;
5911   if (! token.roleId.empty()) {
5912     RGWRole role(s->cct, ctl, token.roleId);
5913     if (role.get_by_id() < 0) {
5914       return result_t::deny(-EPERM);
5915     }
5916     r.id = token.roleId;
5917     r.name = role.get_name();
5918     r.tenant = role.get_tenant();
5919
5920     vector<string> role_policy_names = role.get_role_policy_names();
5921     for (auto& policy_name : role_policy_names) {
5922       string perm_policy;
5923       if (int ret = role.get_role_policy(policy_name, perm_policy); ret == 0) {
5924         r.role_policies.push_back(std::move(perm_policy));
5925       }
5926     }
5927     // This is mostly needed to assign the owner of a bucket during its creation
5928     user_id = token.user;
5929   }
5930
5931   if (! token.user.empty() && token.acct_type != TYPE_ROLE) {
5932     // get user info
5933     int ret = rgw_get_user_info_by_uid(ctl->user, token.user, user_info, NULL);
5934     if (ret < 0) {
5935       ldpp_dout(dpp, 5) << "ERROR: failed reading user info: uid=" << token.user << dendl;
5936       return result_t::reject(-EPERM);
5937     }
5938   }
5939
5940   if (token.acct_type == TYPE_KEYSTONE || token.acct_type == TYPE_LDAP) {
5941     auto apl = remote_apl_factory->create_apl_remote(cct, s, get_acl_strategy(),
5942                                             get_creds_info(token));
5943     return result_t::grant(std::move(apl), completer_factory(boost::none));
5944   } else if (token.acct_type == TYPE_ROLE) {
5945     auto apl = role_apl_factory->create_apl_role(cct, s, r, user_id, token.policy, token.role_session, token.token_claims);
5946     return result_t::grant(std::move(apl), completer_factory(token.secret_access_key));
5947   } else { // This is for all local users of type TYPE_RGW or TYPE_NONE
5948     string subuser;
5949     auto apl = local_apl_factory->create_apl_local(cct, s, user_info, subuser, token.perm_mask);
5950     return result_t::grant(std::move(apl), completer_factory(token.secret_access_key));
5951   }
5952 }
5953
5954 bool rgw::auth::s3::S3AnonymousEngine::is_applicable(
5955   const req_state* s
5956 ) const noexcept {
5957   if (s->op == OP_OPTIONS) {
5958     return true;
5959   }
5960
5961   AwsVersion version;
5962   AwsRoute route;
5963   std::tie(version, route) = discover_aws_flavour(s->info);
5964
5965   return route == AwsRoute::QUERY_STRING && version == AwsVersion::UNKNOWN;
5966 }