ceph/src/rgw/rgw_rest_s3.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3
   4 #include <errno.h>
   5 #include <array>
   6 #include <string.h>
   7
   8 #include "common/ceph_crypto.h"
   9 #include "common/Formatter.h"
  10 #include "common/utf8.h"
  11 #include "common/ceph_json.h"
  12 #include "common/safe_io.h"
  13 #include "common/backport14.h"
  14 #include <boost/algorithm/string.hpp>
  15 #include <boost/algorithm/string/replace.hpp>
  16 #include <boost/utility/string_view.hpp>
  17
  18 #include "rgw_rest.h"
  19 #include "rgw_rest_s3.h"
  20 #include "rgw_rest_s3website.h"
  21 #include "rgw_auth_s3.h"
  22 #include "rgw_acl.h"
  23 #include "rgw_policy_s3.h"
  24 #include "rgw_user.h"
  25 #include "rgw_cors.h"
  26 #include "rgw_cors_s3.h"
  27 #include "rgw_tag_s3.h"
  28
  29 #include "rgw_client_io.h"
  30
  31 #include "rgw_keystone.h"
  32 #include "rgw_auth_keystone.h"
  33 #include "rgw_auth_registry.h"
  34
  35 #include "rgw_es_query.h"
  36
  37 #include <typeinfo> // for 'typeid'
  38
  39 #include "rgw_ldap.h"
  40 #include "rgw_token.h"
  41 #include "rgw_rest_role.h"
  42 #include "rgw_crypt.h"
  43 #include "rgw_crypt_sanitize.h"
  44
  45 #include "include/assert.h"
  46
  47 #define dout_context g_ceph_context
  48 #define dout_subsys ceph_subsys_rgw
  49
  50 using namespace rgw;
  51 using namespace ceph::crypto;
  52
  53 using std::get;
  54
  55 void list_all_buckets_start(struct req_state *s)
  56 {
  57   s->formatter->open_array_section_in_ns("ListAllMyBucketsResult", XMLNS_AWS_S3);
  58 }
  59
  60 void list_all_buckets_end(struct req_state *s)
  61 {
  62   s->formatter->close_section();
  63 }
  64
  65 void dump_bucket(struct req_state *s, RGWBucketEnt& obj)
  66 {
  67   s->formatter->open_object_section("Bucket");
  68   s->formatter->dump_string("Name", obj.bucket.name);
  69   dump_time(s, "CreationDate", &obj.creation_time);
  70   s->formatter->close_section();
  71 }
  72
  73 void rgw_get_errno_s3(rgw_http_error *e , int err_no)
  74 {
  75   rgw_http_errors::const_iterator r = rgw_http_s3_errors.find(err_no);
  76
  77   if (r != rgw_http_s3_errors.end()) {
  78     e->http_ret = r->second.first;
  79     e->s3_code = r->second.second;
  80   } else {
  81     e->http_ret = 500;
  82     e->s3_code = "UnknownError";
  83   }
  84 }
  85
  86 struct response_attr_param {
  87   const char *param;
  88   const char *http_attr;
  89 };
  90
  91 static struct response_attr_param resp_attr_params[] = {
  92   {"response-content-type", "Content-Type"},
  93   {"response-content-language", "Content-Language"},
  94   {"response-expires", "Expires"},
  95   {"response-cache-control", "Cache-Control"},
  96   {"response-content-disposition", "Content-Disposition"},
  97   {"response-content-encoding", "Content-Encoding"},
  98   {NULL, NULL},
  99 };
 100
 101 int RGWGetObj_ObjStore_S3Website::send_response_data(bufferlist& bl, off_t bl_ofs, off_t bl_len) {
 102   map<string, bufferlist>::iterator iter;
 103   iter = attrs.find(RGW_ATTR_AMZ_WEBSITE_REDIRECT_LOCATION);
 104   if (iter != attrs.end()) {
 105     bufferlist &bl = iter->second;
 106     s->redirect = string(bl.c_str(), bl.length());
 107     s->err.http_ret = 301;
 108     ldout(s->cct, 20) << __CEPH_ASSERT_FUNCTION << " redirecting per x-amz-website-redirect-location=" << s->redirect << dendl;
 109     op_ret = -ERR_WEBSITE_REDIRECT;
 110     set_req_state_err(s, op_ret);
 111     dump_errno(s);
 112     dump_content_length(s, 0);
 113     dump_redirect(s, s->redirect);
 114     end_header(s, this);
 115     return op_ret;
 116   } else {
 117     return RGWGetObj_ObjStore_S3::send_response_data(bl, bl_ofs, bl_len);
 118   }
 119 }
 120
 121 int RGWGetObj_ObjStore_S3Website::send_response_data_error()
 122 {
 123   return RGWGetObj_ObjStore_S3::send_response_data_error();
 124 }
 125
 126 int RGWGetObj_ObjStore_S3::get_params()
 127 {
 128   // for multisite sync requests, only read the slo manifest itself, rather than
 129   // all of the data from its parts. the parts will sync as separate objects
 130   skip_manifest = s->info.args.exists(RGW_SYS_PARAM_PREFIX "sync-manifest");
 131
 132   // multisite sync requests should fetch encrypted data, along with the
 133   // attributes needed to support decryption on the other zone
 134   if (s->system_request) {
 135     skip_decrypt = s->info.args.exists(RGW_SYS_PARAM_PREFIX "skip-decrypt");
 136   }
 137
 138   return RGWGetObj_ObjStore::get_params();
 139 }
 140
 141 int RGWGetObj_ObjStore_S3::send_response_data_error()
 142 {
 143   bufferlist bl;
 144   return send_response_data(bl, 0 , 0);
 145 }
 146
 147 template <class T>
 148 int decode_attr_bl_single_value(map<string, bufferlist>& attrs, const char *attr_name, T *result, T def_val)
 149 {
 150   map<string, bufferlist>::iterator iter = attrs.find(attr_name);
 151   if (iter == attrs.end()) {
 152     *result = def_val;
 153     return 0;
 154   }
 155   bufferlist& bl = iter->second;
 156   if (bl.length() == 0) {
 157     *result = def_val;
 158     return 0;
 159   }
 160   bufferlist::iterator bliter = bl.begin();
 161   try {
 162     ::decode(*result, bliter);
 163   } catch (buffer::error& err) {
 164     return -EIO;
 165   }
 166   return 0;
 167 }
 168
 169 int RGWGetObj_ObjStore_S3::send_response_data(bufferlist& bl, off_t bl_ofs,
 170                                               off_t bl_len)
 171 {
 172   const char *content_type = NULL;
 173   string content_type_str;
 174   map<string, string> response_attrs;
 175   map<string, string>::iterator riter;
 176   bufferlist metadata_bl;
 177
 178   if (sent_header)
 179     goto send_data;
 180
 181   if (custom_http_ret) {
 182     set_req_state_err(s, 0);
 183     dump_errno(s, custom_http_ret);
 184   } else {
 185     set_req_state_err(s, (partial_content && !op_ret) ? STATUS_PARTIAL_CONTENT
 186                   : op_ret);
 187     dump_errno(s);
 188   }
 189
 190   if (op_ret)
 191     goto done;
 192
 193   if (range_str)
 194     dump_range(s, start, end, s->obj_size);
 195
 196   if (s->system_request &&
 197       s->info.args.exists(RGW_SYS_PARAM_PREFIX "prepend-metadata")) {
 198
 199     dump_header(s, "Rgwx-Object-Size", (long long)total_len);
 200
 201     if (rgwx_stat) {
 202       /*
 203        * in this case, we're not returning the object's content, only the prepended
 204        * extra metadata
 205        */
 206       total_len = 0;
 207     }
 208
 209     /* JSON encode object metadata */
 210     JSONFormatter jf;
 211     jf.open_object_section("obj_metadata");
 212     encode_json("attrs", attrs, &jf);
 213     utime_t ut(lastmod);
 214     encode_json("mtime", ut, &jf);
 215     jf.close_section();
 216     stringstream ss;
 217     jf.flush(ss);
 218     metadata_bl.append(ss.str());
 219     dump_header(s, "Rgwx-Embedded-Metadata-Len", metadata_bl.length());
 220     total_len += metadata_bl.length();
 221   }
 222
 223   if (s->system_request && !real_clock::is_zero(lastmod)) {
 224     /* we end up dumping mtime in two different methods, a bit redundant */
 225     dump_epoch_header(s, "Rgwx-Mtime", lastmod);
 226     uint64_t pg_ver = 0;
 227     int r = decode_attr_bl_single_value(attrs, RGW_ATTR_PG_VER, &pg_ver, (uint64_t)0);
 228     if (r < 0) {
 229       ldout(s->cct, 0) << "ERROR: failed to decode pg ver attr, ignoring" << dendl;
 230     }
 231     dump_header(s, "Rgwx-Obj-PG-Ver", pg_ver);
 232
 233     uint32_t source_zone_short_id = 0;
 234     r = decode_attr_bl_single_value(attrs, RGW_ATTR_SOURCE_ZONE, &source_zone_short_id, (uint32_t)0);
 235     if (r < 0) {
 236       ldout(s->cct, 0) << "ERROR: failed to decode pg ver attr, ignoring" << dendl;
 237     }
 238     if (source_zone_short_id != 0) {
 239       dump_header(s, "Rgwx-Source-Zone-Short-Id", source_zone_short_id);
 240     }
 241   }
 242
 243   for (auto &it : crypt_http_responses)
 244     dump_header(s, it.first, it.second);
 245
 246   dump_content_length(s, total_len);
 247   dump_last_modified(s, lastmod);
 248   if (!version_id.empty()) {
 249     dump_header(s, "x-amz-version-id", version_id);
 250   }
 251
 252
 253   if (! op_ret) {
 254     if (! lo_etag.empty()) {
 255       /* Handle etag of Swift API's large objects (DLO/SLO). It's entirerly
 256        * legit to perform GET on them through S3 API. In such situation,
 257        * a client should receive the composited content with corresponding
 258        * etag value. */
 259       dump_etag(s, lo_etag);
 260     } else {
 261       auto iter = attrs.find(RGW_ATTR_ETAG);
 262       if (iter != attrs.end()) {
 263         dump_etag(s, iter->second);
 264       }
 265     }
 266
 267     for (struct response_attr_param *p = resp_attr_params; p->param; p++) {
 268       bool exists;
 269       string val = s->info.args.get(p->param, &exists);
 270       if (exists) {
 271         if (strcmp(p->param, "response-content-type") != 0) {
 272           response_attrs[p->http_attr] = val;
 273         } else {
 274           content_type_str = val;
 275           content_type = content_type_str.c_str();
 276         }
 277       }
 278     }
 279
 280     for (auto iter = attrs.begin(); iter != attrs.end(); ++iter) {
 281       const char *name = iter->first.c_str();
 282       map<string, string>::iterator aiter = rgw_to_http_attrs.find(name);
 283       if (aiter != rgw_to_http_attrs.end()) {
 284         if (response_attrs.count(aiter->second) == 0) {
 285           /* Was not already overridden by a response param. */
 286           response_attrs[aiter->second] = iter->second.c_str();
 287         }
 288       } else if (iter->first.compare(RGW_ATTR_CONTENT_TYPE) == 0) {
 289         /* Special handling for content_type. */
 290         if (!content_type) {
 291           content_type = iter->second.c_str();
 292         }
 293       } else if (strcmp(name, RGW_ATTR_SLO_UINDICATOR) == 0) {
 294         // this attr has an extra length prefix from ::encode() in prior versions
 295         dump_header(s, "X-Object-Meta-Static-Large-Object", "True");
 296       } else if (strncmp(name, RGW_ATTR_META_PREFIX,
 297                          sizeof(RGW_ATTR_META_PREFIX)-1) == 0) {
 298         /* User custom metadata. */
 299         name += sizeof(RGW_ATTR_PREFIX) - 1;
 300         dump_header(s, name, iter->second);
 301       } else if (iter->first.compare(RGW_ATTR_TAGS) == 0) {
 302         RGWObjTags obj_tags;
 303         try{
 304           bufferlist::iterator it = iter->second.begin();
 305           obj_tags.decode(it);
 306         } catch (buffer::error &err) {
 307           ldout(s->cct,0) << "Error caught buffer::error couldn't decode TagSet " << dendl;
 308         }
 309         dump_header(s, RGW_AMZ_TAG_COUNT, obj_tags.count());
 310       }
 311     }
 312   }
 313
 314 done:
 315   for (riter = response_attrs.begin(); riter != response_attrs.end();
 316        ++riter) {
 317     dump_header(s, riter->first, riter->second);
 318   }
 319
 320   if (op_ret == -ERR_NOT_MODIFIED) {
 321       end_header(s, this);
 322   } else {
 323       if (!content_type)
 324           content_type = "binary/octet-stream";
 325
 326       end_header(s, this, content_type);
 327   }
 328
 329   if (metadata_bl.length()) {
 330     dump_body(s, metadata_bl);
 331   }
 332   sent_header = true;
 333
 334 send_data:
 335   if (get_data && !op_ret) {
 336     int r = dump_body(s, bl.c_str() + bl_ofs, bl_len);
 337     if (r < 0)
 338       return r;
 339   }
 340
 341   return 0;
 342 }
 343
 344 int RGWGetObj_ObjStore_S3::get_decrypt_filter(std::unique_ptr<RGWGetDataCB> *filter, RGWGetDataCB* cb, bufferlist* manifest_bl)
 345 {
 346   if (skip_decrypt) { // bypass decryption for multisite sync requests
 347     return 0;
 348   }
 349
 350   int res = 0;
 351   std::unique_ptr<BlockCrypt> block_crypt;
 352   res = rgw_s3_prepare_decrypt(s, attrs, &block_crypt, crypt_http_responses);
 353   if (res == 0) {
 354     if (block_crypt != nullptr) {
 355       auto f = std::unique_ptr<RGWGetObj_BlockDecrypt>(new RGWGetObj_BlockDecrypt(s->cct, cb, std::move(block_crypt)));
 356       //RGWGetObj_BlockDecrypt* f = new RGWGetObj_BlockDecrypt(s->cct, cb, std::move(block_crypt));
 357       if (f != nullptr) {
 358         if (manifest_bl != nullptr) {
 359           res = f->read_manifest(*manifest_bl);
 360           if (res == 0) {
 361             *filter = std::move(f);
 362           }
 363         }
 364       }
 365     }
 366   }
 367   return res;
 368 }
 369
 370 void RGWGetObjTags_ObjStore_S3::send_response_data(bufferlist& bl)
 371 {
 372   dump_errno(s);
 373   end_header(s, this, "application/xml");
 374   dump_start(s);
 375
 376   s->formatter->open_object_section_in_ns("Tagging", XMLNS_AWS_S3);
 377   s->formatter->open_object_section("TagSet");
 378   if (has_tags){
 379     RGWObjTagSet_S3 tagset;
 380     bufferlist::iterator iter = bl.begin();
 381     try {
 382       tagset.decode(iter);
 383     } catch (buffer::error& err) {
 384       ldout(s->cct,0) << "ERROR: caught buffer::error, couldn't decode TagSet" << dendl;
 385       op_ret= -EIO;
 386       return;
 387     }
 388     tagset.dump_xml(s->formatter);
 389   }
 390   s->formatter->close_section();
 391   s->formatter->close_section();
 392   rgw_flush_formatter_and_reset(s, s->formatter);
 393 }
 394
 395
 396 int RGWPutObjTags_ObjStore_S3::get_params()
 397 {
 398   RGWObjTagsXMLParser parser;
 399
 400   if (!parser.init()){
 401     return -EINVAL;
 402   }
 403
 404   char *data=nullptr;
 405   int len=0;
 406
 407   const auto max_size = s->cct->_conf->rgw_max_put_param_size;
 408   int r = rgw_rest_read_all_input(s, &data, &len, max_size, false);
 409
 410   if (r < 0)
 411     return r;
 412
 413   auto data_deleter = std::unique_ptr<char, decltype(free)*>{data, free};
 414
 415   if (!parser.parse(data, len, 1)) {
 416     return -ERR_MALFORMED_XML;
 417   }
 418
 419   RGWObjTagSet_S3 *obj_tags_s3;
 420   RGWObjTagging_S3 *tagging;
 421
 422   tagging = static_cast<RGWObjTagging_S3 *>(parser.find_first("Tagging"));
 423   obj_tags_s3 = static_cast<RGWObjTagSet_S3 *>(tagging->find_first("TagSet"));
 424   if(!obj_tags_s3){
 425     return -ERR_MALFORMED_XML;
 426   }
 427
 428   RGWObjTags obj_tags;
 429   r = obj_tags_s3->rebuild(obj_tags);
 430   if (r < 0)
 431     return r;
 432
 433   obj_tags.encode(tags_bl);
 434   ldout(s->cct, 20) << "Read " << obj_tags.count() << "tags" << dendl;
 435
 436   return 0;
 437 }
 438
 439 void RGWPutObjTags_ObjStore_S3::send_response()
 440 {
 441   if (op_ret)
 442     set_req_state_err(s, op_ret);
 443   dump_errno(s);
 444   end_header(s, this, "application/xml");
 445   dump_start(s);
 446
 447 }
 448
 449 void RGWDeleteObjTags_ObjStore_S3::send_response()
 450 {
 451   int r = op_ret;
 452   if (r == -ENOENT)
 453     r = 0;
 454   if (!r)
 455     r = STATUS_NO_CONTENT;
 456
 457   set_req_state_err(s, r);
 458   dump_errno(s);
 459   end_header(s, this);
 460 }
 461
 462 void RGWListBuckets_ObjStore_S3::send_response_begin(bool has_buckets)
 463 {
 464   if (op_ret)
 465     set_req_state_err(s, op_ret);
 466   dump_errno(s);
 467   dump_start(s);
 468   end_header(s, NULL, "application/xml");
 469
 470   if (! op_ret) {
 471     list_all_buckets_start(s);
 472     dump_owner(s, s->user->user_id, s->user->display_name);
 473     s->formatter->open_array_section("Buckets");
 474     sent_data = true;
 475   }
 476 }
 477
 478 void RGWListBuckets_ObjStore_S3::send_response_data(RGWUserBuckets& buckets)
 479 {
 480   if (!sent_data)
 481     return;
 482
 483   map<string, RGWBucketEnt>& m = buckets.get_buckets();
 484   map<string, RGWBucketEnt>::iterator iter;
 485
 486   for (iter = m.begin(); iter != m.end(); ++iter) {
 487     RGWBucketEnt obj = iter->second;
 488     dump_bucket(s, obj);
 489   }
 490   rgw_flush_formatter(s, s->formatter);
 491 }
 492
 493 void RGWListBuckets_ObjStore_S3::send_response_end()
 494 {
 495   if (sent_data) {
 496     s->formatter->close_section();
 497     list_all_buckets_end(s);
 498     rgw_flush_formatter_and_reset(s, s->formatter);
 499   }
 500 }
 501
 502 int RGWGetUsage_ObjStore_S3::get_params()
 503 {
 504   start_date = s->info.args.get("start-date");
 505   end_date = s->info.args.get("end-date");
 506   return 0;
 507 }
 508
 509 static void dump_usage_categories_info(Formatter *formatter, const rgw_usage_log_entry& entry, map<string, bool> *categories)
 510 {
 511   formatter->open_array_section("categories");
 512   map<string, rgw_usage_data>::const_iterator uiter;
 513   for (uiter = entry.usage_map.begin(); uiter != entry.usage_map.end(); ++uiter) {
 514     if (categories && !categories->empty() && !categories->count(uiter->first))
 515       continue;
 516     const rgw_usage_data& usage = uiter->second;
 517     formatter->open_object_section("Entry");
 518     formatter->dump_string("Category", uiter->first);
 519     formatter->dump_int("BytesSent", usage.bytes_sent);
 520     formatter->dump_int("BytesReceived", usage.bytes_received);
 521     formatter->dump_int("Ops", usage.ops);
 522     formatter->dump_int("SuccessfulOps", usage.successful_ops);
 523     formatter->close_section(); // Entry
 524   }
 525   formatter->close_section(); // Category
 526 }
 527
 528 static void dump_usage_bucket_info(Formatter *formatter, const std::string& name, const cls_user_bucket_entry& entry)
 529 {
 530   formatter->open_object_section("Entry");
 531   formatter->dump_string("Bucket", name);
 532   formatter->dump_int("Bytes", entry.size);
 533   formatter->dump_int("Bytes_Rounded", entry.size_rounded);
 534   formatter->close_section(); // entry
 535 }
 536
 537 void RGWGetUsage_ObjStore_S3::send_response()
 538 {
 539   if (op_ret < 0)
 540     set_req_state_err(s, op_ret);
 541   dump_errno(s);
 542
 543   end_header(s, this, "application/xml");
 544   dump_start(s);
 545   if (op_ret < 0)
 546     return;
 547
 548   Formatter *formatter = s->formatter;
 549   string last_owner;
 550   bool user_section_open = false;
 551
 552   formatter->open_object_section("Usage");
 553   if (show_log_entries) {
 554     formatter->open_array_section("Entries");
 555   }
 556   map<rgw_user_bucket, rgw_usage_log_entry>::iterator iter;
 557   for (iter = usage.begin(); iter != usage.end(); ++iter) {
 558     const rgw_user_bucket& ub = iter->first;
 559     const rgw_usage_log_entry& entry = iter->second;
 560
 561     if (show_log_entries) {
 562       if (ub.user.compare(last_owner) != 0) {
 563         if (user_section_open) {
 564           formatter->close_section();
 565           formatter->close_section();
 566         }
 567         formatter->open_object_section("User");
 568         formatter->dump_string("Owner", ub.user);
 569         formatter->open_array_section("Buckets");
 570         user_section_open = true;
 571         last_owner = ub.user;
 572       }
 573       formatter->open_object_section("Bucket");
 574       formatter->dump_string("Bucket", ub.bucket);
 575       utime_t ut(entry.epoch, 0);
 576       ut.gmtime(formatter->dump_stream("Time"));
 577       formatter->dump_int("Epoch", entry.epoch);
 578       dump_usage_categories_info(formatter, entry, &categories);
 579       formatter->close_section(); // bucket
 580     }
 581
 582     summary_map[ub.user].aggregate(entry, &categories);
 583   }
 584
 585   if (show_log_entries) {
 586      if (user_section_open) {
 587        formatter->close_section(); // buckets
 588        formatter->close_section(); //user
 589      }
 590      formatter->close_section(); // entries
 591    }
 592
 593    if (show_log_sum) {
 594      formatter->open_array_section("Summary");
 595      map<string, rgw_usage_log_entry>::iterator siter;
 596      for (siter = summary_map.begin(); siter != summary_map.end(); ++siter) {
 597        const rgw_usage_log_entry& entry = siter->second;
 598        formatter->open_object_section("User");
 599        formatter->dump_string("User", siter->first);
 600        dump_usage_categories_info(formatter, entry, &categories);
 601        rgw_usage_data total_usage;
 602        entry.sum(total_usage, categories);
 603        formatter->open_object_section("Total");
 604        formatter->dump_int("BytesSent", total_usage.bytes_sent);
 605        formatter->dump_int("BytesReceived", total_usage.bytes_received);
 606        formatter->dump_int("Ops", total_usage.ops);
 607        formatter->dump_int("SuccessfulOps", total_usage.successful_ops);
 608        formatter->close_section(); // total
 609        formatter->close_section(); // user
 610      }
 611
 612      if (s->cct->_conf->rgw_rest_getusage_op_compat) {
 613        formatter->open_object_section("Stats");
 614      }
 615
 616      formatter->dump_int("TotalBytes", header.stats.total_bytes);
 617      formatter->dump_int("TotalBytesRounded", header.stats.total_bytes_rounded);
 618      formatter->dump_int("TotalEntries", header.stats.total_entries);
 619
 620      if (s->cct->_conf->rgw_rest_getusage_op_compat) {
 621        formatter->close_section(); //Stats
 622      }
 623
 624      formatter->close_section(); // summary
 625    }
 626
 627   formatter->open_array_section("CapacityUsed");
 628   formatter->open_object_section("User");
 629   formatter->open_array_section("Buckets");
 630   for (const auto& biter : buckets_usage) {
 631     const cls_user_bucket_entry& entry = biter.second;
 632     dump_usage_bucket_info(formatter, biter.first, entry);
 633   }
 634   formatter->close_section(); // Buckets
 635   formatter->close_section(); // User
 636   formatter->close_section(); // CapacityUsed
 637
 638   formatter->close_section(); // usage
 639   rgw_flush_formatter_and_reset(s, s->formatter);
 640 }
 641
 642 int RGWListBucket_ObjStore_S3::get_params()
 643 {
 644   list_versions = s->info.args.exists("versions");
 645   prefix = s->info.args.get("prefix");
 646   if (!list_versions) {
 647     marker = s->info.args.get("marker");
 648   } else {
 649     marker.name = s->info.args.get("key-marker");
 650     marker.instance = s->info.args.get("version-id-marker");
 651   }
 652
 653   // non-standard
 654   s->info.args.get_bool("allow-unordered", &allow_unordered, false);
 655
 656   delimiter = s->info.args.get("delimiter");
 657
 658   max_keys = s->info.args.get("max-keys");
 659   op_ret = parse_max_keys();
 660   if (op_ret < 0) {
 661     return op_ret;
 662   }
 663
 664   encoding_type = s->info.args.get("encoding-type");
 665   if (s->system_request) {
 666     s->info.args.get_bool("objs-container", &objs_container, false);
 667     const char *shard_id_str = s->info.env->get("HTTP_RGWX_SHARD_ID");
 668     if (shard_id_str) {
 669       string err;
 670       shard_id = strict_strtol(shard_id_str, 10, &err);
 671       if (!err.empty()) {
 672         ldout(s->cct, 5) << "bad shard id specified: " << shard_id_str << dendl;
 673         return -EINVAL;
 674       }
 675     } else {
 676       shard_id = s->bucket_instance_shard_id;
 677     }
 678   }
 679
 680   return 0;
 681 }
 682
 683 void RGWListBucket_ObjStore_S3::send_versioned_response()
 684 {
 685   s->formatter->open_object_section_in_ns("ListVersionsResult", XMLNS_AWS_S3);
 686   if (!s->bucket_tenant.empty())
 687     s->formatter->dump_string("Tenant", s->bucket_tenant);
 688   s->formatter->dump_string("Name", s->bucket_name);
 689   s->formatter->dump_string("Prefix", prefix);
 690   s->formatter->dump_string("KeyMarker", marker.name);
 691   s->formatter->dump_string("VersionIdMarker", marker.instance);
 692   if (is_truncated && !next_marker.empty()) {
 693     s->formatter->dump_string("NextKeyMarker", next_marker.name);
 694     s->formatter->dump_string("NextVersionIdMarker", next_marker.instance);
 695   }
 696   s->formatter->dump_int("MaxKeys", max);
 697   if (!delimiter.empty())
 698     s->formatter->dump_string("Delimiter", delimiter);
 699
 700   s->formatter->dump_string("IsTruncated", (max && is_truncated ? "true"
 701                                             : "false"));
 702
 703   bool encode_key = false;
 704   if (strcasecmp(encoding_type.c_str(), "url") == 0) {
 705     s->formatter->dump_string("EncodingType", "url");
 706     encode_key = true;
 707   }
 708
 709   if (op_ret >= 0) {
 710     if (objs_container) {
 711       s->formatter->open_array_section("Entries");
 712     }
 713
 714     vector<rgw_bucket_dir_entry>::iterator iter;
 715     for (iter = objs.begin(); iter != objs.end(); ++iter) {
 716       const char *section_name = (iter->is_delete_marker() ? "DeleteMarker"
 717                                   : "Version");
 718       s->formatter->open_object_section(section_name);
 719       if (objs_container) {
 720         s->formatter->dump_bool("IsDeleteMarker", iter->is_delete_marker());
 721       }
 722       rgw_obj_key key(iter->key);
 723       if (encode_key) {
 724         string key_name;
 725         url_encode(key.name, key_name);
 726         s->formatter->dump_string("Key", key_name);
 727       } else {
 728         s->formatter->dump_string("Key", key.name);
 729       }
 730       string version_id = key.instance;
 731       if (version_id.empty()) {
 732         version_id = "null";
 733       }
 734       if (s->system_request) {
 735         if (iter->versioned_epoch > 0) {
 736           s->formatter->dump_int("VersionedEpoch", iter->versioned_epoch);
 737         }
 738         s->formatter->dump_string("RgwxTag", iter->tag);
 739         utime_t ut(iter->meta.mtime);
 740         ut.gmtime_nsec(s->formatter->dump_stream("RgwxMtime"));
 741       }
 742       s->formatter->dump_string("VersionId", version_id);
 743       s->formatter->dump_bool("IsLatest", iter->is_current());
 744       dump_time(s, "LastModified", &iter->meta.mtime);
 745       if (!iter->is_delete_marker()) {
 746         s->formatter->dump_format("ETag", "\"%s\"", iter->meta.etag.c_str());
 747         s->formatter->dump_int("Size", iter->meta.accounted_size);
 748         s->formatter->dump_string("StorageClass", "STANDARD");
 749       }
 750       dump_owner(s, iter->meta.owner, iter->meta.owner_display_name);
 751       s->formatter->close_section();
 752     }
 753     if (objs_container) {
 754       s->formatter->close_section();
 755     }
 756
 757     if (!common_prefixes.empty()) {
 758       map<string, bool>::iterator pref_iter;
 759       for (pref_iter = common_prefixes.begin();
 760            pref_iter != common_prefixes.end(); ++pref_iter) {
 761         s->formatter->open_array_section("CommonPrefixes");
 762         s->formatter->dump_string("Prefix", pref_iter->first);
 763         s->formatter->close_section();
 764       }
 765     }
 766   }
 767   s->formatter->close_section();
 768   rgw_flush_formatter_and_reset(s, s->formatter);
 769 }
 770
 771 void RGWListBucket_ObjStore_S3::send_response()
 772 {
 773   if (op_ret < 0)
 774     set_req_state_err(s, op_ret);
 775   dump_errno(s);
 776
 777   end_header(s, this, "application/xml");
 778   dump_start(s);
 779   if (op_ret < 0)
 780     return;
 781
 782   if (list_versions) {
 783     send_versioned_response();
 784     return;
 785   }
 786
 787   s->formatter->open_object_section_in_ns("ListBucketResult", XMLNS_AWS_S3);
 788   if (!s->bucket_tenant.empty())
 789     s->formatter->dump_string("Tenant", s->bucket_tenant);
 790   s->formatter->dump_string("Name", s->bucket_name);
 791   s->formatter->dump_string("Prefix", prefix);
 792   s->formatter->dump_string("Marker", marker.name);
 793   if (is_truncated && !next_marker.empty())
 794     s->formatter->dump_string("NextMarker", next_marker.name);
 795   s->formatter->dump_int("MaxKeys", max);
 796   if (!delimiter.empty())
 797     s->formatter->dump_string("Delimiter", delimiter);
 798
 799   s->formatter->dump_string("IsTruncated", (max && is_truncated ? "true"
 800                                             : "false"));
 801
 802   bool encode_key = false;
 803   if (strcasecmp(encoding_type.c_str(), "url") == 0) {
 804     s->formatter->dump_string("EncodingType", "url");
 805     encode_key = true;
 806   }
 807
 808   if (op_ret >= 0) {
 809     vector<rgw_bucket_dir_entry>::iterator iter;
 810     for (iter = objs.begin(); iter != objs.end(); ++iter) {
 811       rgw_obj_key key(iter->key);
 812       s->formatter->open_array_section("Contents");
 813       if (encode_key) {
 814         string key_name;
 815         url_encode(key.name, key_name);
 816         s->formatter->dump_string("Key", key_name);
 817       } else {
 818         s->formatter->dump_string("Key", key.name);
 819       }
 820       dump_time(s, "LastModified", &iter->meta.mtime);
 821       s->formatter->dump_format("ETag", "\"%s\"", iter->meta.etag.c_str());
 822       s->formatter->dump_int("Size", iter->meta.accounted_size);
 823       s->formatter->dump_string("StorageClass", "STANDARD");
 824       dump_owner(s, iter->meta.owner, iter->meta.owner_display_name);
 825       if (s->system_request) {
 826         s->formatter->dump_string("RgwxTag", iter->tag);
 827       }
 828       s->formatter->close_section();
 829     }
 830     if (!common_prefixes.empty()) {
 831       map<string, bool>::iterator pref_iter;
 832       for (pref_iter = common_prefixes.begin();
 833            pref_iter != common_prefixes.end(); ++pref_iter) {
 834         s->formatter->open_array_section("CommonPrefixes");
 835         s->formatter->dump_string("Prefix", pref_iter->first);
 836         s->formatter->close_section();
 837       }
 838     }
 839   }
 840   s->formatter->close_section();
 841   rgw_flush_formatter_and_reset(s, s->formatter);
 842 }
 843
 844 void RGWGetBucketLogging_ObjStore_S3::send_response()
 845 {
 846   dump_errno(s);
 847   end_header(s, this, "application/xml");
 848   dump_start(s);
 849
 850   s->formatter->open_object_section_in_ns("BucketLoggingStatus", XMLNS_AWS_S3);
 851   s->formatter->close_section();
 852   rgw_flush_formatter_and_reset(s, s->formatter);
 853 }
 854
 855 void RGWGetBucketLocation_ObjStore_S3::send_response()
 856 {
 857   dump_errno(s);
 858   end_header(s, this);
 859   dump_start(s);
 860
 861   RGWZoneGroup zonegroup;
 862   string api_name;
 863
 864   int ret = store->get_zonegroup(s->bucket_info.zonegroup, zonegroup);
 865   if (ret >= 0) {
 866     api_name = zonegroup.api_name;
 867   } else  {
 868     if (s->bucket_info.zonegroup != "default") {
 869       api_name = s->bucket_info.zonegroup;
 870     }
 871   }
 872
 873   s->formatter->dump_format_ns("LocationConstraint", XMLNS_AWS_S3,
 874                                "%s", api_name.c_str());
 875   rgw_flush_formatter_and_reset(s, s->formatter);
 876 }
 877
 878 void RGWGetBucketVersioning_ObjStore_S3::send_response()
 879 {
 880   dump_errno(s);
 881   end_header(s, this, "application/xml");
 882   dump_start(s);
 883
 884   s->formatter->open_object_section_in_ns("VersioningConfiguration", XMLNS_AWS_S3);
 885   if (versioned) {
 886     const char *status = (versioning_enabled ? "Enabled" : "Suspended");
 887     s->formatter->dump_string("Status", status);
 888   }
 889   s->formatter->close_section();
 890   rgw_flush_formatter_and_reset(s, s->formatter);
 891 }
 892
 893 class RGWSetBucketVersioningParser : public RGWXMLParser
 894 {
 895   XMLObj *alloc_obj(const char *el) override {
 896     return new XMLObj;
 897   }
 898
 899 public:
 900   RGWSetBucketVersioningParser() {}
 901   ~RGWSetBucketVersioningParser() override {}
 902
 903   int get_versioning_status(bool *status) {
 904     XMLObj *config = find_first("VersioningConfiguration");
 905     if (!config)
 906       return -EINVAL;
 907
 908     *status = false;
 909
 910     XMLObj *field = config->find_first("Status");
 911     if (!field)
 912       return 0;
 913
 914     string& s = field->get_data();
 915
 916     if (stringcasecmp(s, "Enabled") == 0) {
 917       *status = true;
 918     } else if (stringcasecmp(s, "Suspended") != 0) {
 919       return -EINVAL;
 920     }
 921
 922     return 0;
 923   }
 924 };
 925
 926 int RGWSetBucketVersioning_ObjStore_S3::get_params()
 927 {
 928   char *data = nullptr;
 929   int len = 0;
 930   int r =
 931     rgw_rest_read_all_input(s, &data, &len, s->cct->_conf->rgw_max_put_param_size, false);
 932   if (r < 0) {
 933     return r;
 934   }
 935
 936   auto data_deleter = std::unique_ptr<char, decltype(free)*>{data, free};
 937
 938   r = do_aws4_auth_completion();
 939   if (r < 0) {
 940     return r;
 941   }
 942
 943   RGWSetBucketVersioningParser parser;
 944
 945   if (!parser.init()) {
 946     ldout(s->cct, 0) << "ERROR: failed to initialize parser" << dendl;
 947     r = -EIO;
 948     return r;
 949   }
 950
 951   if (!parser.parse(data, len, 1)) {
 952     ldout(s->cct, 10) << "failed to parse data: " << data << dendl;
 953     r = -EINVAL;
 954     return r;
 955   }
 956
 957   if (!store->is_meta_master()) {
 958     /* only need to keep this data around if we're not meta master */
 959     in_data.append(data, len);
 960   }
 961
 962   r = parser.get_versioning_status(&enable_versioning);
 963
 964   return r;
 965 }
 966
 967 void RGWSetBucketVersioning_ObjStore_S3::send_response()
 968 {
 969   if (op_ret)
 970     set_req_state_err(s, op_ret);
 971   dump_errno(s);
 972   end_header(s);
 973 }
 974
 975 int RGWSetBucketWebsite_ObjStore_S3::get_params()
 976 {
 977   char *data = nullptr;
 978   int len = 0;
 979   const auto max_size = s->cct->_conf->rgw_max_put_param_size;
 980   int r = rgw_rest_read_all_input(s, &data, &len, max_size, false);
 981
 982   if (r < 0) {
 983     return r;
 984   }
 985
 986   auto data_deleter = std::unique_ptr<char, decltype(free)*>{data, free};
 987
 988   r = do_aws4_auth_completion();
 989   if (r < 0) {
 990     return r;
 991   }
 992
 993   bufferptr in_ptr(data, len);
 994   in_data.append(in_ptr);
 995
 996   RGWXMLDecoder::XMLParser parser;
 997   if (!parser.init()) {
 998     ldout(s->cct, 0) << "ERROR: failed to initialize parser" << dendl;
 999     return -EIO;
1000   }
1001
1002   if (!parser.parse(data, len, 1)) {
1003     string str(data, len);
1004     ldout(s->cct, 5) << "failed to parse xml: " << str << dendl;
1005     return -EINVAL;
1006   }
1007
1008   try {
1009     RGWXMLDecoder::decode_xml("WebsiteConfiguration", website_conf, &parser, true);
1010   } catch (RGWXMLDecoder::err& err) {
1011     string str(data, len);
1012     ldout(s->cct, 5) << "unexpected xml: " << str << dendl;
1013     return -EINVAL;
1014   }
1015
1016   return 0;
1017 }
1018
1019 void RGWSetBucketWebsite_ObjStore_S3::send_response()
1020 {
1021   if (op_ret < 0)
1022     set_req_state_err(s, op_ret);
1023   dump_errno(s);
1024   end_header(s);
1025 }
1026
1027 void RGWDeleteBucketWebsite_ObjStore_S3::send_response()
1028 {
1029   if (op_ret == 0) {
1030     op_ret = STATUS_NO_CONTENT;
1031   }
1032   set_req_state_err(s, op_ret);
1033   dump_errno(s);
1034   end_header(s);
1035 }
1036
1037 void RGWGetBucketWebsite_ObjStore_S3::send_response()
1038 {
1039   if (op_ret)
1040     set_req_state_err(s, op_ret);
1041   dump_errno(s);
1042   end_header(s, this, "application/xml");
1043   dump_start(s);
1044
1045   if (op_ret < 0) {
1046     return;
1047   }
1048
1049   RGWBucketWebsiteConf& conf = s->bucket_info.website_conf;
1050
1051   s->formatter->open_object_section_in_ns("WebsiteConfiguration", XMLNS_AWS_S3);
1052   conf.dump_xml(s->formatter);
1053   s->formatter->close_section(); // WebsiteConfiguration
1054   rgw_flush_formatter_and_reset(s, s->formatter);
1055 }
1056
1057 static void dump_bucket_metadata(struct req_state *s, RGWBucketEnt& bucket)
1058 {
1059   dump_header(s, "X-RGW-Object-Count", static_cast<long long>(bucket.count));
1060   dump_header(s, "X-RGW-Bytes-Used", static_cast<long long>(bucket.size));
1061 }
1062
1063 void RGWStatBucket_ObjStore_S3::send_response()
1064 {
1065   if (op_ret >= 0) {
1066     dump_bucket_metadata(s, bucket);
1067   }
1068
1069   set_req_state_err(s, op_ret);
1070   dump_errno(s);
1071
1072   end_header(s, this);
1073   dump_start(s);
1074 }
1075
1076 static int create_s3_policy(struct req_state *s, RGWRados *store,
1077                             RGWAccessControlPolicy_S3& s3policy,
1078                             ACLOwner& owner)
1079 {
1080   if (s->has_acl_header) {
1081     if (!s->canned_acl.empty())
1082       return -ERR_INVALID_REQUEST;
1083
1084     return s3policy.create_from_headers(store, s->info.env, owner);
1085   }
1086
1087   return s3policy.create_canned(owner, s->bucket_owner, s->canned_acl);
1088 }
1089
1090 class RGWLocationConstraint : public XMLObj
1091 {
1092 public:
1093   RGWLocationConstraint() {}
1094   ~RGWLocationConstraint() override {}
1095   bool xml_end(const char *el) override {
1096     if (!el)
1097       return false;
1098
1099     location_constraint = get_data();
1100
1101     return true;
1102   }
1103
1104   string location_constraint;
1105 };
1106
1107 class RGWCreateBucketConfig : public XMLObj
1108 {
1109 public:
1110   RGWCreateBucketConfig() {}
1111   ~RGWCreateBucketConfig() override {}
1112 };
1113
1114 class RGWCreateBucketParser : public RGWXMLParser
1115 {
1116   XMLObj *alloc_obj(const char *el) override {
1117     return new XMLObj;
1118   }
1119
1120 public:
1121   RGWCreateBucketParser() {}
1122   ~RGWCreateBucketParser() override {}
1123
1124   bool get_location_constraint(string& zone_group) {
1125     XMLObj *config = find_first("CreateBucketConfiguration");
1126     if (!config)
1127       return false;
1128
1129     XMLObj *constraint = config->find_first("LocationConstraint");
1130     if (!constraint)
1131       return false;
1132
1133     zone_group = constraint->get_data();
1134
1135     return true;
1136   }
1137 };
1138
1139 int RGWCreateBucket_ObjStore_S3::get_params()
1140 {
1141   RGWAccessControlPolicy_S3 s3policy(s->cct);
1142
1143   int r = create_s3_policy(s, store, s3policy, s->owner);
1144   if (r < 0)
1145     return r;
1146
1147   policy = s3policy;
1148
1149   int len = 0;
1150   char *data = nullptr;
1151
1152   const auto max_size = s->cct->_conf->rgw_max_put_param_size;
1153   op_ret = rgw_rest_read_all_input(s, &data, &len, max_size, false);
1154
1155   if ((op_ret < 0) && (op_ret != -ERR_LENGTH_REQUIRED))
1156     return op_ret;
1157
1158   auto data_deleter = std::unique_ptr<char, decltype(free)*>{data, free};
1159
1160   const int auth_ret = do_aws4_auth_completion();
1161   if (auth_ret < 0) {
1162     return auth_ret;
1163   }
1164
1165   bufferptr in_ptr(data, len);
1166   in_data.append(in_ptr);
1167
1168   if (len) {
1169     RGWCreateBucketParser parser;
1170
1171     if (!parser.init()) {
1172       ldout(s->cct, 0) << "ERROR: failed to initialize parser" << dendl;
1173       return -EIO;
1174     }
1175
1176     bool success = parser.parse(data, len, 1);
1177     ldout(s->cct, 20) << "create bucket input data=" << data << dendl;
1178
1179     if (!success) {
1180       ldout(s->cct, 0) << "failed to parse input: " << data << dendl;
1181       return -EINVAL;
1182     }
1183
1184     if (!parser.get_location_constraint(location_constraint)) {
1185       ldout(s->cct, 0) << "provided input did not specify location constraint correctly" << dendl;
1186       return -EINVAL;
1187     }
1188
1189     ldout(s->cct, 10) << "create bucket location constraint: "
1190                       << location_constraint << dendl;
1191   }
1192
1193   size_t pos = location_constraint.find(':');
1194   if (pos != string::npos) {
1195     placement_rule = location_constraint.substr(pos + 1);
1196     location_constraint = location_constraint.substr(0, pos);
1197   }
1198
1199   return 0;
1200 }
1201
1202 void RGWCreateBucket_ObjStore_S3::send_response()
1203 {
1204   if (op_ret == -ERR_BUCKET_EXISTS)
1205     op_ret = 0;
1206   if (op_ret)
1207     set_req_state_err(s, op_ret);
1208   dump_errno(s);
1209   end_header(s);
1210
1211   if (op_ret < 0)
1212     return;
1213
1214   if (s->system_request) {
1215     JSONFormatter f; /* use json formatter for system requests output */
1216
1217     f.open_object_section("info");
1218     encode_json("entry_point_object_ver", ep_objv, &f);
1219     encode_json("object_ver", info.objv_tracker.read_version, &f);
1220     encode_json("bucket_info", info, &f);
1221     f.close_section();
1222     rgw_flush_formatter_and_reset(s, &f);
1223   }
1224 }
1225
1226 void RGWDeleteBucket_ObjStore_S3::send_response()
1227 {
1228   int r = op_ret;
1229   if (!r)
1230     r = STATUS_NO_CONTENT;
1231
1232   set_req_state_err(s, r);
1233   dump_errno(s);
1234   end_header(s, this);
1235
1236   if (s->system_request) {
1237     JSONFormatter f; /* use json formatter for system requests output */
1238
1239     f.open_object_section("info");
1240     encode_json("object_ver", objv_tracker.read_version, &f);
1241     f.close_section();
1242     rgw_flush_formatter_and_reset(s, &f);
1243   }
1244 }
1245
1246 int RGWPutObj_ObjStore_S3::get_params()
1247 {
1248   if (!s->length)
1249     return -ERR_LENGTH_REQUIRED;
1250
1251   RGWObjectCtx& obj_ctx = *static_cast<RGWObjectCtx *>(s->obj_ctx);
1252   map<string, bufferlist> src_attrs;
1253   size_t pos;
1254   int ret;
1255
1256   RGWAccessControlPolicy_S3 s3policy(s->cct);
1257   ret = create_s3_policy(s, store, s3policy, s->owner);
1258   if (ret < 0)
1259     return ret;
1260
1261   policy = s3policy;
1262
1263   if_match = s->info.env->get("HTTP_IF_MATCH");
1264   if_nomatch = s->info.env->get("HTTP_IF_NONE_MATCH");
1265   copy_source = url_decode(s->info.env->get("HTTP_X_AMZ_COPY_SOURCE", ""));
1266   copy_source_range = s->info.env->get("HTTP_X_AMZ_COPY_SOURCE_RANGE");
1267
1268   /* handle x-amz-copy-source */
1269   boost::string_view cs_view(copy_source);
1270   if (! cs_view.empty()) {
1271     if (cs_view[0] == '/')
1272       cs_view.remove_prefix(1);
1273     copy_source_bucket_name = cs_view.to_string();
1274     pos = copy_source_bucket_name.find("/");
1275     if (pos == std::string::npos) {
1276       ret = -EINVAL;
1277       ldout(s->cct, 5) << "x-amz-copy-source bad format" << dendl;
1278       return ret;
1279     }
1280     copy_source_object_name =
1281       copy_source_bucket_name.substr(pos + 1, copy_source_bucket_name.size());
1282     copy_source_bucket_name = copy_source_bucket_name.substr(0, pos);
1283 #define VERSION_ID_STR "?versionId="
1284     pos = copy_source_object_name.find(VERSION_ID_STR);
1285     if (pos == std::string::npos) {
1286       copy_source_object_name = url_decode(copy_source_object_name);
1287     } else {
1288       copy_source_version_id =
1289         copy_source_object_name.substr(pos + sizeof(VERSION_ID_STR) - 1);
1290       copy_source_object_name =
1291         url_decode(copy_source_object_name.substr(0, pos));
1292     }
1293     pos = copy_source_bucket_name.find(":");
1294     if (pos == std::string::npos) {
1295        copy_source_tenant_name = s->src_tenant_name;
1296     } else {
1297        copy_source_tenant_name = copy_source_bucket_name.substr(0, pos);
1298        copy_source_bucket_name = copy_source_bucket_name.substr(pos + 1, copy_source_bucket_name.size());
1299        if (copy_source_bucket_name.empty()) {
1300          ret = -EINVAL;
1301          ldout(s->cct, 5) << "source bucket name is empty" << dendl;
1302          return ret;
1303        }
1304     }
1305     ret = store->get_bucket_info(obj_ctx,
1306                                  copy_source_tenant_name,
1307                                  copy_source_bucket_name,
1308                                  copy_source_bucket_info,
1309                                  NULL, &src_attrs);
1310     if (ret < 0) {
1311       ldout(s->cct, 5) << __func__ << "(): get_bucket_info() returned ret=" << ret << dendl;
1312       return ret;
1313     }
1314
1315     /* handle x-amz-copy-source-range */
1316
1317     if (copy_source_range) {
1318       string range = copy_source_range;
1319       pos = range.find("=");
1320       if (pos == std::string::npos) {
1321         ret = -EINVAL;
1322         ldout(s->cct, 5) << "x-amz-copy-source-range bad format" << dendl;
1323         return ret;
1324       }
1325       range = range.substr(pos + 1);
1326       pos = range.find("-");
1327       if (pos == std::string::npos) {
1328         ret = -EINVAL;
1329         ldout(s->cct, 5) << "x-amz-copy-source-range bad format" << dendl;
1330         return ret;
1331       }
1332       string first = range.substr(0, pos);
1333       string last = range.substr(pos + 1);
1334       copy_source_range_fst = strtoull(first.c_str(), NULL, 10);
1335       copy_source_range_lst = strtoull(last.c_str(), NULL, 10);
1336     }
1337
1338   } /* copy_source */
1339
1340   /* handle object tagging */
1341   auto tag_str = s->info.env->get("HTTP_X_AMZ_TAGGING");
1342   if (tag_str){
1343     obj_tags = ceph::make_unique<RGWObjTags>();
1344     ret = obj_tags->set_from_string(tag_str);
1345     if (ret < 0){
1346       ldout(s->cct,0) << "setting obj tags failed with " << ret << dendl;
1347       if (ret == -ERR_INVALID_TAG){
1348         ret = -EINVAL; //s3 returns only -EINVAL for PUT requests
1349       }
1350
1351       return ret;
1352     }
1353   }
1354
1355   return RGWPutObj_ObjStore::get_params();
1356 }
1357
1358 int RGWPutObj_ObjStore_S3::get_data(bufferlist& bl)
1359 {
1360   const int ret = RGWPutObj_ObjStore::get_data(bl);
1361   if (ret == 0) {
1362     const int ret_auth = do_aws4_auth_completion();
1363     if (ret_auth < 0) {
1364       return ret_auth;
1365     }
1366   }
1367
1368   return ret;
1369 }
1370
1371 static int get_success_retcode(int code)
1372 {
1373   switch (code) {
1374     case 201:
1375       return STATUS_CREATED;
1376     case 204:
1377       return STATUS_NO_CONTENT;
1378   }
1379   return 0;
1380 }
1381
1382 void RGWPutObj_ObjStore_S3::send_response()
1383 {
1384   if (op_ret) {
1385     set_req_state_err(s, op_ret);
1386     dump_errno(s);
1387   } else {
1388     if (s->cct->_conf->rgw_s3_success_create_obj_status) {
1389       op_ret = get_success_retcode(
1390         s->cct->_conf->rgw_s3_success_create_obj_status);
1391       set_req_state_err(s, op_ret);
1392     }
1393     if (copy_source.empty()) {
1394       dump_errno(s);
1395       dump_etag(s, etag);
1396       dump_content_length(s, 0);
1397       for (auto &it : crypt_http_responses)
1398         dump_header(s, it.first, it.second);
1399     } else {
1400       dump_errno(s);
1401       end_header(s, this, "application/xml");
1402       dump_start(s);
1403       struct tm tmp;
1404       utime_t ut(mtime);
1405       time_t secs = (time_t)ut.sec();
1406       gmtime_r(&secs, &tmp);
1407       char buf[TIME_BUF_SIZE];
1408       s->formatter->open_object_section_in_ns("CopyPartResult",
1409           "http://s3.amazonaws.com/doc/2006-03-01/");
1410       if (strftime(buf, sizeof(buf), "%Y-%m-%dT%T.000Z", &tmp) > 0) {
1411         s->formatter->dump_string("LastModified", buf);
1412       }
1413       s->formatter->dump_string("ETag", etag);
1414       s->formatter->close_section();
1415       rgw_flush_formatter_and_reset(s, s->formatter);
1416       return;
1417     }
1418   }
1419   if (s->system_request && !real_clock::is_zero(mtime)) {
1420     dump_epoch_header(s, "Rgwx-Mtime", mtime);
1421   }
1422   end_header(s, this);
1423 }
1424
1425 static inline int get_obj_attrs(RGWRados *store, struct req_state *s, rgw_obj& obj, map<string, bufferlist>& attrs)
1426 {
1427   RGWRados::Object op_target(store, s->bucket_info, *static_cast<RGWObjectCtx *>(s->obj_ctx), obj);
1428   RGWRados::Object::Read read_op(&op_target);
1429
1430   read_op.params.attrs = &attrs;
1431
1432   return read_op.prepare();
1433 }
1434
1435 static inline void set_attr(map<string, bufferlist>& attrs, const char* key, const std::string& value)
1436 {
1437   bufferlist bl;
1438   ::encode(value,bl);
1439   attrs.emplace(key, std::move(bl));
1440 }
1441
1442 static inline void set_attr(map<string, bufferlist>& attrs, const char* key, const char* value)
1443 {
1444   bufferlist bl;
1445   ::encode(value,bl);
1446   attrs.emplace(key, std::move(bl));
1447 }
1448
1449 int RGWPutObj_ObjStore_S3::get_decrypt_filter(
1450     std::unique_ptr<RGWGetDataCB>* filter,
1451     RGWGetDataCB* cb,
1452     map<string, bufferlist>& attrs,
1453     bufferlist* manifest_bl)
1454 {
1455   std::map<std::string, std::string> crypt_http_responses_unused;
1456
1457   int res = 0;
1458   std::unique_ptr<BlockCrypt> block_crypt;
1459   res = rgw_s3_prepare_decrypt(s, attrs, &block_crypt, crypt_http_responses_unused);
1460   if (res == 0) {
1461     if (block_crypt != nullptr) {
1462       auto f = std::unique_ptr<RGWGetObj_BlockDecrypt>(new RGWGetObj_BlockDecrypt(s->cct, cb, std::move(block_crypt)));
1463       //RGWGetObj_BlockDecrypt* f = new RGWGetObj_BlockDecrypt(s->cct, cb, std::move(block_crypt));
1464       if (f != nullptr) {
1465         if (manifest_bl != nullptr) {
1466           res = f->read_manifest(*manifest_bl);
1467           if (res == 0) {
1468             *filter = std::move(f);
1469           }
1470         }
1471       }
1472     }
1473   }
1474   return res;
1475 }
1476
1477 int RGWPutObj_ObjStore_S3::get_encrypt_filter(
1478     std::unique_ptr<RGWPutObjDataProcessor>* filter,
1479     RGWPutObjDataProcessor* cb)
1480 {
1481   int res = 0;
1482   RGWPutObjProcessor_Multipart* multi_processor=dynamic_cast<RGWPutObjProcessor_Multipart*>(cb);
1483   if (multi_processor != nullptr) {
1484     RGWMPObj* mp = nullptr;
1485     multi_processor->get_mp(&mp);
1486     if (mp != nullptr) {
1487       map<string, bufferlist> xattrs;
1488       string meta_oid;
1489       meta_oid = mp->get_meta();
1490
1491       rgw_obj obj;
1492       obj.init_ns(s->bucket, meta_oid, RGW_OBJ_NS_MULTIPART);
1493       obj.set_in_extra_data(true);
1494       res = get_obj_attrs(store, s, obj, xattrs);
1495       if (res == 0) {
1496         std::unique_ptr<BlockCrypt> block_crypt;
1497         /* We are adding to existing object.
1498          * We use crypto mode that configured as if we were decrypting. */
1499         res = rgw_s3_prepare_decrypt(s, xattrs, &block_crypt, crypt_http_responses);
1500         if (res == 0 && block_crypt != nullptr)
1501           *filter = std::unique_ptr<RGWPutObj_BlockEncrypt>(
1502               new RGWPutObj_BlockEncrypt(s->cct, cb, std::move(block_crypt)));
1503       }
1504     }
1505     /* it is ok, to not have encryption at all */
1506   }
1507   else
1508   {
1509     std::unique_ptr<BlockCrypt> block_crypt;
1510     res = rgw_s3_prepare_encrypt(s, attrs, nullptr, &block_crypt, crypt_http_responses);
1511     if (res == 0 && block_crypt != nullptr) {
1512       *filter = std::unique_ptr<RGWPutObj_BlockEncrypt>(
1513           new RGWPutObj_BlockEncrypt(s->cct, cb, std::move(block_crypt)));
1514     }
1515   }
1516   return res;
1517 }
1518
1519 void RGWPostObj_ObjStore_S3::rebuild_key(string& key)
1520 {
1521   static string var = "${filename}";
1522   int pos = key.find(var);
1523   if (pos < 0)
1524     return;
1525
1526   string new_key = key.substr(0, pos);
1527   new_key.append(filename);
1528   new_key.append(key.substr(pos + var.size()));
1529
1530   key = new_key;
1531 }
1532
1533 std::string RGWPostObj_ObjStore_S3::get_current_filename() const
1534 {
1535   return s->object.name;
1536 }
1537
1538 std::string RGWPostObj_ObjStore_S3::get_current_content_type() const
1539 {
1540   return content_type;
1541 }
1542
1543 int RGWPostObj_ObjStore_S3::get_params()
1544 {
1545   op_ret = RGWPostObj_ObjStore::get_params();
1546   if (op_ret < 0) {
1547     return op_ret;
1548   }
1549
1550   ldout(s->cct, 20) << "adding bucket to policy env: " << s->bucket.name
1551                     << dendl;
1552   env.add_var("bucket", s->bucket.name);
1553
1554   bool done;
1555   do {
1556     struct post_form_part part;
1557     int r = read_form_part_header(&part, done);
1558     if (r < 0)
1559       return r;
1560
1561     if (s->cct->_conf->subsys.should_gather(ceph_subsys_rgw, 20)) {
1562       ldout(s->cct, 20) << "read part header -- part.name="
1563                         << part.name << dendl;
1564
1565       for (const auto& pair : part.fields) {
1566         ldout(s->cct, 20) << "field.name=" << pair.first << dendl;
1567         ldout(s->cct, 20) << "field.val=" << pair.second.val << dendl;
1568         ldout(s->cct, 20) << "field.params:" << dendl;
1569
1570         for (const auto& param_pair : pair.second.params) {
1571           ldout(s->cct, 20) << " " << param_pair.first
1572                             << " -> " << param_pair.second << dendl;
1573         }
1574       }
1575     }
1576
1577     if (done) { /* unexpected here */
1578       err_msg = "Malformed request";
1579       return -EINVAL;
1580     }
1581
1582     if (stringcasecmp(part.name, "file") == 0) { /* beginning of data transfer */
1583       struct post_part_field& field = part.fields["Content-Disposition"];
1584       map<string, string>::iterator iter = field.params.find("filename");
1585       if (iter != field.params.end()) {
1586         filename = iter->second;
1587       }
1588       parts[part.name] = part;
1589       break;
1590     }
1591
1592     bool boundary;
1593     uint64_t chunk_size = s->cct->_conf->rgw_max_chunk_size;
1594     r = read_data(part.data, chunk_size, boundary, done);
1595     if (r < 0 || !boundary) {
1596       err_msg = "Couldn't find boundary";
1597       return -EINVAL;
1598     }
1599     parts[part.name] = part;
1600     string part_str(part.data.c_str(), part.data.length());
1601     env.add_var(part.name, part_str);
1602   } while (!done);
1603
1604   string object_str;
1605   if (!part_str(parts, "key", &object_str)) {
1606     err_msg = "Key not specified";
1607     return -EINVAL;
1608   }
1609
1610   s->object = rgw_obj_key(object_str);
1611
1612   rebuild_key(s->object.name);
1613
1614   if (s->object.empty()) {
1615     err_msg = "Empty object name";
1616     return -EINVAL;
1617   }
1618
1619   env.add_var("key", s->object.name);
1620
1621   part_str(parts, "Content-Type", &content_type);
1622
1623   /* AWS permits POST without Content-Type: http://tracker.ceph.com/issues/20201 */
1624   if (! content_type.empty()) {
1625     env.add_var("Content-Type", content_type);
1626   }
1627
1628   map<string, struct post_form_part, ltstr_nocase>::iterator piter =
1629     parts.upper_bound(RGW_AMZ_META_PREFIX);
1630   for (; piter != parts.end(); ++piter) {
1631     string n = piter->first;
1632     if (strncasecmp(n.c_str(), RGW_AMZ_META_PREFIX,
1633                     sizeof(RGW_AMZ_META_PREFIX) - 1) != 0)
1634       break;
1635
1636     string attr_name = RGW_ATTR_PREFIX;
1637     attr_name.append(n);
1638
1639     /* need to null terminate it */
1640     bufferlist& data = piter->second.data;
1641     string str = string(data.c_str(), data.length());
1642
1643     bufferlist attr_bl;
1644     attr_bl.append(str.c_str(), str.size() + 1);
1645
1646     attrs[attr_name] = attr_bl;
1647   }
1648   // TODO: refactor this and the above loop to share code
1649   piter = parts.find(RGW_AMZ_WEBSITE_REDIRECT_LOCATION);
1650   if (piter != parts.end()) {
1651     string n = piter->first;
1652     string attr_name = RGW_ATTR_PREFIX;
1653     attr_name.append(n);
1654     /* need to null terminate it */
1655     bufferlist& data = piter->second.data;
1656     string str = string(data.c_str(), data.length());
1657
1658     bufferlist attr_bl;
1659     attr_bl.append(str.c_str(), str.size() + 1);
1660
1661     attrs[attr_name] = attr_bl;
1662   }
1663
1664   int r = get_policy();
1665   if (r < 0)
1666     return r;
1667
1668   r = get_tags();
1669   if (r < 0)
1670     return r;
1671
1672
1673   min_len = post_policy.min_length;
1674   max_len = post_policy.max_length;
1675
1676
1677
1678   return 0;
1679 }
1680
1681 int RGWPostObj_ObjStore_S3::get_tags()
1682 {
1683   string tags_str;
1684   if (part_str(parts, "tagging", &tags_str)) {
1685     RGWObjTagsXMLParser parser;
1686     if (!parser.init()){
1687       ldout(s->cct, 0) << "Couldn't init RGWObjTags XML parser" << dendl;
1688       err_msg = "Server couldn't process the request";
1689       return -EINVAL; // TODO: This class of errors in rgw code should be a 5XX error
1690     }
1691     if (!parser.parse(tags_str.c_str(), tags_str.size(), 1)) {
1692       ldout(s->cct,0 ) << "Invalid Tagging XML" << dendl;
1693       err_msg = "Invalid Tagging XML";
1694       return -EINVAL;
1695     }
1696
1697     RGWObjTagSet_S3 *obj_tags_s3;
1698     RGWObjTagging_S3 *tagging;
1699
1700     tagging = static_cast<RGWObjTagging_S3 *>(parser.find_first("Tagging"));
1701     obj_tags_s3 = static_cast<RGWObjTagSet_S3 *>(tagging->find_first("TagSet"));
1702     if(!obj_tags_s3){
1703       return -ERR_MALFORMED_XML;
1704     }
1705
1706     RGWObjTags obj_tags;
1707     int r = obj_tags_s3->rebuild(obj_tags);
1708     if (r < 0)
1709       return r;
1710
1711     bufferlist tags_bl;
1712     obj_tags.encode(tags_bl);
1713     ldout(s->cct, 20) << "Read " << obj_tags.count() << "tags" << dendl;
1714     attrs[RGW_ATTR_TAGS] = tags_bl;
1715   }
1716
1717
1718   return 0;
1719 }
1720
1721 int RGWPostObj_ObjStore_S3::get_policy()
1722 {
1723   if (part_bl(parts, "policy", &s->auth.s3_postobj_creds.encoded_policy)) {
1724     bool aws4_auth = false;
1725
1726     /* x-amz-algorithm handling */
1727     using rgw::auth::s3::AWS4_HMAC_SHA256_STR;
1728     if ((part_str(parts, "x-amz-algorithm", &s->auth.s3_postobj_creds.x_amz_algorithm)) &&
1729         (s->auth.s3_postobj_creds.x_amz_algorithm == AWS4_HMAC_SHA256_STR)) {
1730       ldout(s->cct, 0) << "Signature verification algorithm AWS v4 (AWS4-HMAC-SHA256)" << dendl;
1731       aws4_auth = true;
1732     } else {
1733       ldout(s->cct, 0) << "Signature verification algorithm AWS v2" << dendl;
1734     }
1735
1736     // check that the signature matches the encoded policy
1737     if (aws4_auth) {
1738       /* AWS4 */
1739
1740       /* x-amz-credential handling */
1741       if (!part_str(parts, "x-amz-credential",
1742                     &s->auth.s3_postobj_creds.x_amz_credential)) {
1743         ldout(s->cct, 0) << "No S3 aws4 credential found!" << dendl;
1744         err_msg = "Missing aws4 credential";
1745         return -EINVAL;
1746       }
1747
1748       /* x-amz-signature handling */
1749       if (!part_str(parts, "x-amz-signature",
1750                     &s->auth.s3_postobj_creds.signature)) {
1751         ldout(s->cct, 0) << "No aws4 signature found!" << dendl;
1752         err_msg = "Missing aws4 signature";
1753         return -EINVAL;
1754       }
1755
1756       /* x-amz-date handling */
1757       std::string received_date_str;
1758       if (!part_str(parts, "x-amz-date", &received_date_str)) {
1759         ldout(s->cct, 0) << "No aws4 date found!" << dendl;
1760         err_msg = "Missing aws4 date";
1761         return -EINVAL;
1762       }
1763     } else {
1764       /* AWS2 */
1765
1766       // check that the signature matches the encoded policy
1767       if (!part_str(parts, "AWSAccessKeyId",
1768                     &s->auth.s3_postobj_creds.access_key)) {
1769         ldout(s->cct, 0) << "No S3 aws2 access key found!" << dendl;
1770         err_msg = "Missing aws2 access key";
1771         return -EINVAL;
1772       }
1773
1774       if (!part_str(parts, "signature", &s->auth.s3_postobj_creds.signature)) {
1775         ldout(s->cct, 0) << "No aws2 signature found!" << dendl;
1776         err_msg = "Missing aws2 signature";
1777         return -EINVAL;
1778       }
1779     }
1780
1781     /* FIXME: this is a makeshift solution. The browser upload authentication will be
1782      * handled by an instance of rgw::auth::Completer spawned in Handler's authorize()
1783      * method. */
1784     const int ret = rgw::auth::Strategy::apply(auth_registry_ptr->get_s3_post(), s);
1785     if (ret != 0) {
1786       return -EACCES;
1787     } else {
1788       /* Populate the owner info. */
1789       s->owner.set_id(s->user->user_id);
1790       s->owner.set_name(s->user->display_name);
1791       ldout(s->cct, 20) << "Successful Signature Verification!" << dendl;
1792     }
1793
1794     ceph::bufferlist decoded_policy;
1795     try {
1796       decoded_policy.decode_base64(s->auth.s3_postobj_creds.encoded_policy);
1797     } catch (buffer::error& err) {
1798       ldout(s->cct, 0) << "failed to decode_base64 policy" << dendl;
1799       err_msg = "Could not decode policy";
1800       return -EINVAL;
1801     }
1802
1803     decoded_policy.append('\0'); // NULL terminate
1804     ldout(s->cct, 20) << "POST policy: " << decoded_policy.c_str() << dendl;
1805
1806
1807     int r = post_policy.from_json(decoded_policy, err_msg);
1808     if (r < 0) {
1809       if (err_msg.empty()) {
1810         err_msg = "Failed to parse policy";
1811       }
1812       ldout(s->cct, 0) << "failed to parse policy" << dendl;
1813       return -EINVAL;
1814     }
1815
1816     if (aws4_auth) {
1817       /* AWS4 */
1818       post_policy.set_var_checked("x-amz-signature");
1819     } else {
1820       /* AWS2 */
1821       post_policy.set_var_checked("AWSAccessKeyId");
1822       post_policy.set_var_checked("signature");
1823     }
1824     post_policy.set_var_checked("policy");
1825
1826     r = post_policy.check(&env, err_msg);
1827     if (r < 0) {
1828       if (err_msg.empty()) {
1829         err_msg = "Policy check failed";
1830       }
1831       ldout(s->cct, 0) << "policy check failed" << dendl;
1832       return r;
1833     }
1834
1835   } else {
1836     ldout(s->cct, 0) << "No attached policy found!" << dendl;
1837   }
1838
1839   string canned_acl;
1840   part_str(parts, "acl", &canned_acl);
1841
1842   RGWAccessControlPolicy_S3 s3policy(s->cct);
1843   ldout(s->cct, 20) << "canned_acl=" << canned_acl << dendl;
1844   if (s3policy.create_canned(s->owner, s->bucket_owner, canned_acl) < 0) {
1845     err_msg = "Bad canned ACLs";
1846     return -EINVAL;
1847   }
1848
1849   policy = s3policy;
1850
1851   return 0;
1852 }
1853
1854 int RGWPostObj_ObjStore_S3::complete_get_params()
1855 {
1856   bool done;
1857   do {
1858     struct post_form_part part;
1859     int r = read_form_part_header(&part, done);
1860     if (r < 0) {
1861       return r;
1862     }
1863
1864     ceph::bufferlist part_data;
1865     bool boundary;
1866     uint64_t chunk_size = s->cct->_conf->rgw_max_chunk_size;
1867     r = read_data(part.data, chunk_size, boundary, done);
1868     if (r < 0 || !boundary) {
1869       return -EINVAL;
1870     }
1871
1872     /* Just reading the data but not storing any results of that. */
1873   } while (!done);
1874
1875   return 0;
1876 }
1877
1878 int RGWPostObj_ObjStore_S3::get_data(ceph::bufferlist& bl, bool& again)
1879 {
1880   bool boundary;
1881   bool done;
1882
1883   const uint64_t chunk_size = s->cct->_conf->rgw_max_chunk_size;
1884   int r = read_data(bl, chunk_size, boundary, done);
1885   if (r < 0) {
1886     return r;
1887   }
1888
1889   if (boundary) {
1890     if (!done) {
1891       /* Reached end of data, let's drain the rest of the params */
1892       r = complete_get_params();
1893       if (r < 0) {
1894        return r;
1895       }
1896     }
1897   }
1898
1899   again = !boundary;
1900   return bl.length();
1901 }
1902
1903 void RGWPostObj_ObjStore_S3::send_response()
1904 {
1905   if (op_ret == 0 && parts.count("success_action_redirect")) {
1906     string redirect;
1907
1908     part_str(parts, "success_action_redirect", &redirect);
1909
1910     string tenant;
1911     string bucket;
1912     string key;
1913     string etag_str = "\"";
1914
1915     etag_str.append(etag);
1916     etag_str.append("\"");
1917
1918     string etag_url;
1919
1920     url_encode(s->bucket_tenant, tenant); /* surely overkill, but cheap */
1921     url_encode(s->bucket_name, bucket);
1922     url_encode(s->object.name, key);
1923     url_encode(etag_str, etag_url);
1924
1925     if (!s->bucket_tenant.empty()) {
1926       /*
1927        * What we really would like is to quaily the bucket name, so
1928        * that the client could simply copy it and paste into next request.
1929        * Unfortunately, in S3 we cannot know if the client will decide
1930        * to come through DNS, with "bucket.tenant" sytanx, or through
1931        * URL with "tenant\bucket" syntax. Therefore, we provide the
1932        * tenant separately.
1933        */
1934       redirect.append("?tenant=");
1935       redirect.append(tenant);
1936       redirect.append("&bucket=");
1937       redirect.append(bucket);
1938     } else {
1939       redirect.append("?bucket=");
1940       redirect.append(bucket);
1941     }
1942     redirect.append("&key=");
1943     redirect.append(key);
1944     redirect.append("&etag=");
1945     redirect.append(etag_url);
1946
1947     int r = check_utf8(redirect.c_str(), redirect.size());
1948     if (r < 0) {
1949       op_ret = r;
1950       goto done;
1951     }
1952     dump_redirect(s, redirect);
1953     op_ret = STATUS_REDIRECT;
1954   } else if (op_ret == 0 && parts.count("success_action_status")) {
1955     string status_string;
1956     uint32_t status_int;
1957
1958     part_str(parts, "success_action_status", &status_string);
1959
1960     int r = stringtoul(status_string, &status_int);
1961     if (r < 0) {
1962       op_ret = r;
1963       goto done;
1964     }
1965
1966     switch (status_int) {
1967       case 200:
1968         break;
1969       case 201:
1970         op_ret = STATUS_CREATED;
1971         break;
1972       default:
1973         op_ret = STATUS_NO_CONTENT;
1974         break;
1975     }
1976   } else if (! op_ret) {
1977     op_ret = STATUS_NO_CONTENT;
1978   }
1979
1980 done:
1981   if (op_ret == STATUS_CREATED) {
1982     for (auto &it : crypt_http_responses)
1983       dump_header(s, it.first, it.second);
1984     s->formatter->open_object_section("PostResponse");
1985     if (g_conf->rgw_dns_name.length())
1986       s->formatter->dump_format("Location", "%s/%s",
1987                                 s->info.script_uri.c_str(),
1988                                 s->object.name.c_str());
1989     if (!s->bucket_tenant.empty())
1990       s->formatter->dump_string("Tenant", s->bucket_tenant);
1991     s->formatter->dump_string("Bucket", s->bucket_name);
1992     s->formatter->dump_string("Key", s->object.name);
1993     s->formatter->close_section();
1994   }
1995   s->err.message = err_msg;
1996   set_req_state_err(s, op_ret);
1997   dump_errno(s);
1998   if (op_ret >= 0) {
1999     dump_content_length(s, s->formatter->get_len());
2000   }
2001   end_header(s, this);
2002   if (op_ret != STATUS_CREATED)
2003     return;
2004
2005   rgw_flush_formatter_and_reset(s, s->formatter);
2006 }
2007
2008 int RGWPostObj_ObjStore_S3::get_encrypt_filter(
2009     std::unique_ptr<RGWPutObjDataProcessor>* filter, RGWPutObjDataProcessor* cb)
2010 {
2011   int res = 0;
2012   std::unique_ptr<BlockCrypt> block_crypt;
2013   res = rgw_s3_prepare_encrypt(s, attrs, &parts, &block_crypt, crypt_http_responses);
2014   if (res == 0 && block_crypt != nullptr) {
2015     *filter = std::unique_ptr<RGWPutObj_BlockEncrypt>(
2016         new RGWPutObj_BlockEncrypt(s->cct, cb, std::move(block_crypt)));
2017   }
2018   else
2019     *filter = nullptr;
2020   return res;
2021 }
2022
2023 int RGWDeleteObj_ObjStore_S3::get_params()
2024 {
2025   const char *if_unmod = s->info.env->get("HTTP_X_AMZ_DELETE_IF_UNMODIFIED_SINCE");
2026
2027   if (s->system_request) {
2028     s->info.args.get_bool(RGW_SYS_PARAM_PREFIX "no-precondition-error", &no_precondition_error, false);
2029   }
2030
2031   if (if_unmod) {
2032     std::string if_unmod_decoded = url_decode(if_unmod);
2033     uint64_t epoch;
2034     uint64_t nsec;
2035     if (utime_t::parse_date(if_unmod_decoded, &epoch, &nsec) < 0) {
2036       ldout(s->cct, 10) << "failed to parse time: " << if_unmod_decoded << dendl;
2037       return -EINVAL;
2038     }
2039     unmod_since = utime_t(epoch, nsec).to_real_time();
2040   }
2041
2042   return 0;
2043 }
2044
2045 void RGWDeleteObj_ObjStore_S3::send_response()
2046 {
2047   int r = op_ret;
2048   if (r == -ENOENT)
2049     r = 0;
2050   if (!r)
2051     r = STATUS_NO_CONTENT;
2052
2053   set_req_state_err(s, r);
2054   dump_errno(s);
2055   if (!version_id.empty()) {
2056     dump_header(s, "x-amz-version-id", version_id);
2057   }
2058   if (delete_marker) {
2059     dump_header(s, "x-amz-delete-marker", "true");
2060   }
2061   end_header(s, this);
2062 }
2063
2064 int RGWCopyObj_ObjStore_S3::init_dest_policy()
2065 {
2066   RGWAccessControlPolicy_S3 s3policy(s->cct);
2067
2068   /* build a policy for the target object */
2069   int r = create_s3_policy(s, store, s3policy, s->owner);
2070   if (r < 0)
2071     return r;
2072
2073   dest_policy = s3policy;
2074
2075   return 0;
2076 }
2077
2078 int RGWCopyObj_ObjStore_S3::get_params()
2079 {
2080   if_mod = s->info.env->get("HTTP_X_AMZ_COPY_IF_MODIFIED_SINCE");
2081   if_unmod = s->info.env->get("HTTP_X_AMZ_COPY_IF_UNMODIFIED_SINCE");
2082   if_match = s->info.env->get("HTTP_X_AMZ_COPY_IF_MATCH");
2083   if_nomatch = s->info.env->get("HTTP_X_AMZ_COPY_IF_NONE_MATCH");
2084
2085   src_tenant_name = s->src_tenant_name;
2086   src_bucket_name = s->src_bucket_name;
2087   src_object = s->src_object;
2088   dest_tenant_name = s->bucket.tenant;
2089   dest_bucket_name = s->bucket.name;
2090   dest_object = s->object.name;
2091
2092   if (s->system_request) {
2093     source_zone = s->info.args.get(RGW_SYS_PARAM_PREFIX "source-zone");
2094     s->info.args.get_bool(RGW_SYS_PARAM_PREFIX "copy-if-newer", &copy_if_newer, false);
2095     if (!source_zone.empty()) {
2096       client_id = s->info.args.get(RGW_SYS_PARAM_PREFIX "client-id");
2097       op_id = s->info.args.get(RGW_SYS_PARAM_PREFIX "op-id");
2098
2099       if (client_id.empty() || op_id.empty()) {
2100         ldout(s->cct, 0) <<
2101           RGW_SYS_PARAM_PREFIX "client-id or "
2102           RGW_SYS_PARAM_PREFIX "op-id were not provided, "
2103           "required for intra-region copy"
2104                          << dendl;
2105         return -EINVAL;
2106       }
2107     }
2108   }
2109
2110   const char *md_directive = s->info.env->get("HTTP_X_AMZ_METADATA_DIRECTIVE");
2111   if (md_directive) {
2112     if (strcasecmp(md_directive, "COPY") == 0) {
2113       attrs_mod = RGWRados::ATTRSMOD_NONE;
2114     } else if (strcasecmp(md_directive, "REPLACE") == 0) {
2115       attrs_mod = RGWRados::ATTRSMOD_REPLACE;
2116     } else if (!source_zone.empty()) {
2117       attrs_mod = RGWRados::ATTRSMOD_NONE; // default for intra-zone_group copy
2118     } else {
2119       ldout(s->cct, 0) << "invalid metadata directive" << dendl;
2120       return -EINVAL;
2121     }
2122   }
2123
2124   if (source_zone.empty() &&
2125       (dest_tenant_name.compare(src_tenant_name) == 0) &&
2126       (dest_bucket_name.compare(src_bucket_name) == 0) &&
2127       (dest_object.compare(src_object.name) == 0) &&
2128       src_object.instance.empty() &&
2129       (attrs_mod != RGWRados::ATTRSMOD_REPLACE)) {
2130     /* can only copy object into itself if replacing attrs */
2131     ldout(s->cct, 0) << "can't copy object into itself if not replacing attrs"
2132                      << dendl;
2133     return -ERR_INVALID_REQUEST;
2134   }
2135   return 0;
2136 }
2137
2138 void RGWCopyObj_ObjStore_S3::send_partial_response(off_t ofs)
2139 {
2140   if (! sent_header) {
2141     if (op_ret)
2142     set_req_state_err(s, op_ret);
2143     dump_errno(s);
2144
2145     end_header(s, this, "application/xml");
2146     dump_start(s);
2147     if (op_ret == 0) {
2148       s->formatter->open_object_section_in_ns("CopyObjectResult", XMLNS_AWS_S3);
2149     }
2150     sent_header = true;
2151   } else {
2152     /* Send progress field. Note that this diverge from the original S3
2153      * spec. We do this in order to keep connection alive.
2154      */
2155     s->formatter->dump_int("Progress", (uint64_t)ofs);
2156   }
2157   rgw_flush_formatter(s, s->formatter);
2158 }
2159
2160 void RGWCopyObj_ObjStore_S3::send_response()
2161 {
2162   if (!sent_header)
2163     send_partial_response(0);
2164
2165   if (op_ret == 0) {
2166     dump_time(s, "LastModified", &mtime);
2167     std::string etag_str = etag.to_str();
2168     if (! etag_str.empty()) {
2169       s->formatter->dump_string("ETag", std::move(etag_str));
2170     }
2171     s->formatter->close_section();
2172     rgw_flush_formatter_and_reset(s, s->formatter);
2173   }
2174 }
2175
2176 void RGWGetACLs_ObjStore_S3::send_response()
2177 {
2178   if (op_ret)
2179     set_req_state_err(s, op_ret);
2180   dump_errno(s);
2181   end_header(s, this, "application/xml");
2182   dump_start(s);
2183   rgw_flush_formatter(s, s->formatter);
2184   dump_body(s, acls);
2185 }
2186
2187 int RGWPutACLs_ObjStore_S3::get_params()
2188 {
2189   int ret =  RGWPutACLs_ObjStore::get_params();
2190   if (ret >= 0) {
2191     const int ret_auth = do_aws4_auth_completion();
2192     if (ret_auth < 0) {
2193       return ret_auth;
2194     }
2195   }
2196   return ret;
2197 }
2198
2199 int RGWPutACLs_ObjStore_S3::get_policy_from_state(RGWRados *store,
2200                                                   struct req_state *s,
2201                                                   stringstream& ss)
2202 {
2203   RGWAccessControlPolicy_S3 s3policy(s->cct);
2204
2205   // bucket-* canned acls do not apply to bucket
2206   if (s->object.empty()) {
2207     if (s->canned_acl.find("bucket") != string::npos)
2208       s->canned_acl.clear();
2209   }
2210
2211   int r = create_s3_policy(s, store, s3policy, owner);
2212   if (r < 0)
2213     return r;
2214
2215   s3policy.to_xml(ss);
2216
2217   return 0;
2218 }
2219
2220 void RGWPutACLs_ObjStore_S3::send_response()
2221 {
2222   if (op_ret)
2223     set_req_state_err(s, op_ret);
2224   dump_errno(s);
2225   end_header(s, this, "application/xml");
2226   dump_start(s);
2227 }
2228
2229 void RGWGetLC_ObjStore_S3::execute()
2230 {
2231   config.set_ctx(s->cct);
2232
2233   map<string, bufferlist>::iterator aiter = s->bucket_attrs.find(RGW_ATTR_LC);
2234   if (aiter == s->bucket_attrs.end()) {
2235     op_ret = -ENOENT;
2236     return;
2237   }
2238
2239   bufferlist::iterator iter(&aiter->second);
2240   try {
2241       config.decode(iter);
2242     } catch (const buffer::error& e) {
2243       ldout(s->cct, 0) << __func__ <<  "decode life cycle config failed" << dendl;
2244       op_ret = -EIO;
2245       return;
2246     }
2247 }
2248
2249 void RGWGetLC_ObjStore_S3::send_response()
2250 {
2251   if (op_ret) {
2252     if (op_ret == -ENOENT) {
2253       set_req_state_err(s, ERR_NO_SUCH_LC);
2254     } else {
2255       set_req_state_err(s, op_ret);
2256     }
2257   }
2258   dump_errno(s);
2259   end_header(s, this, "application/xml");
2260   dump_start(s);
2261
2262   if (op_ret < 0)
2263     return;
2264
2265   config.dump_xml(s->formatter);
2266   rgw_flush_formatter_and_reset(s, s->formatter);
2267 }
2268
2269 void RGWPutLC_ObjStore_S3::send_response()
2270 {
2271   if (op_ret)
2272     set_req_state_err(s, op_ret);
2273   dump_errno(s);
2274   end_header(s, this, "application/xml");
2275   dump_start(s);
2276 }
2277
2278 void RGWDeleteLC_ObjStore_S3::send_response()
2279 {
2280   if (op_ret == 0)
2281       op_ret = STATUS_NO_CONTENT;
2282   if (op_ret) {
2283     set_req_state_err(s, op_ret);
2284   }
2285   dump_errno(s);
2286   end_header(s, this, "application/xml");
2287   dump_start(s);
2288 }
2289
2290 void RGWGetCORS_ObjStore_S3::send_response()
2291 {
2292   if (op_ret) {
2293     if (op_ret == -ENOENT)
2294       set_req_state_err(s, ERR_NO_SUCH_CORS_CONFIGURATION);
2295     else
2296       set_req_state_err(s, op_ret);
2297   }
2298   dump_errno(s);
2299   end_header(s, NULL, "application/xml");
2300   dump_start(s);
2301   if (! op_ret) {
2302     string cors;
2303     RGWCORSConfiguration_S3 *s3cors =
2304       static_cast<RGWCORSConfiguration_S3 *>(&bucket_cors);
2305     stringstream ss;
2306
2307     s3cors->to_xml(ss);
2308     cors = ss.str();
2309     dump_body(s, cors);
2310   }
2311 }
2312
2313 int RGWPutCORS_ObjStore_S3::get_params()
2314 {
2315   int r;
2316   char *data = nullptr;
2317   int len = 0;
2318   RGWCORSXMLParser_S3 parser(s->cct);
2319   RGWCORSConfiguration_S3 *cors_config;
2320
2321   const auto max_size = s->cct->_conf->rgw_max_put_param_size;
2322   r = rgw_rest_read_all_input(s, &data, &len, max_size, false);
2323   if (r < 0) {
2324     return r;
2325   }
2326
2327   auto data_deleter = std::unique_ptr<char, decltype(free)*>{data, free};
2328
2329   r = do_aws4_auth_completion();
2330   if (r < 0) {
2331     return r;
2332   }
2333
2334   if (!parser.init()) {
2335     return -EINVAL;
2336   }
2337
2338   if (!data || !parser.parse(data, len, 1)) {
2339     return -EINVAL;
2340   }
2341   cors_config =
2342     static_cast<RGWCORSConfiguration_S3 *>(parser.find_first(
2343                                              "CORSConfiguration"));
2344   if (!cors_config) {
2345     return -EINVAL;
2346   }
2347
2348   // forward bucket cors requests to meta master zone
2349   if (!store->is_meta_master()) {
2350     /* only need to keep this data around if we're not meta master */
2351     in_data.append(data, len);
2352   }
2353
2354   if (s->cct->_conf->subsys.should_gather(ceph_subsys_rgw, 15)) {
2355     ldout(s->cct, 15) << "CORSConfiguration";
2356     cors_config->to_xml(*_dout);
2357     *_dout << dendl;
2358   }
2359
2360   cors_config->encode(cors_bl);
2361
2362   return 0;
2363 }
2364
2365 void RGWPutCORS_ObjStore_S3::send_response()
2366 {
2367   if (op_ret)
2368     set_req_state_err(s, op_ret);
2369   dump_errno(s);
2370   end_header(s, NULL, "application/xml");
2371   dump_start(s);
2372 }
2373
2374 void RGWDeleteCORS_ObjStore_S3::send_response()
2375 {
2376   int r = op_ret;
2377   if (!r || r == -ENOENT)
2378     r = STATUS_NO_CONTENT;
2379
2380   set_req_state_err(s, r);
2381   dump_errno(s);
2382   end_header(s, NULL);
2383 }
2384
2385 void RGWOptionsCORS_ObjStore_S3::send_response()
2386 {
2387   string hdrs, exp_hdrs;
2388   uint32_t max_age = CORS_MAX_AGE_INVALID;
2389   /*EACCES means, there is no CORS registered yet for the bucket
2390    *ENOENT means, there is no match of the Origin in the list of CORSRule
2391    */
2392   if (op_ret == -ENOENT)
2393     op_ret = -EACCES;
2394   if (op_ret < 0) {
2395     set_req_state_err(s, op_ret);
2396     dump_errno(s);
2397     end_header(s, NULL);
2398     return;
2399   }
2400   get_response_params(hdrs, exp_hdrs, &max_age);
2401
2402   dump_errno(s);
2403   dump_access_control(s, origin, req_meth, hdrs.c_str(), exp_hdrs.c_str(),
2404                       max_age);
2405   end_header(s, NULL);
2406 }
2407
2408 void RGWGetRequestPayment_ObjStore_S3::send_response()
2409 {
2410   dump_errno(s);
2411   end_header(s, this, "application/xml");
2412   dump_start(s);
2413
2414   s->formatter->open_object_section_in_ns("RequestPaymentConfiguration", XMLNS_AWS_S3);
2415   const char *payer = requester_pays ? "Requester" :  "BucketOwner";
2416   s->formatter->dump_string("Payer", payer);
2417   s->formatter->close_section();
2418   rgw_flush_formatter_and_reset(s, s->formatter);
2419 }
2420
2421 class RGWSetRequestPaymentParser : public RGWXMLParser
2422 {
2423   XMLObj *alloc_obj(const char *el) override {
2424     return new XMLObj;
2425   }
2426
2427 public:
2428   RGWSetRequestPaymentParser() {}
2429   ~RGWSetRequestPaymentParser() override {}
2430
2431   int get_request_payment_payer(bool *requester_pays) {
2432     XMLObj *config = find_first("RequestPaymentConfiguration");
2433     if (!config)
2434       return -EINVAL;
2435
2436     *requester_pays = false;
2437
2438     XMLObj *field = config->find_first("Payer");
2439     if (!field)
2440       return 0;
2441
2442     string& s = field->get_data();
2443
2444     if (stringcasecmp(s, "Requester") == 0) {
2445       *requester_pays = true;
2446     } else if (stringcasecmp(s, "BucketOwner") != 0) {
2447       return -EINVAL;
2448     }
2449
2450     return 0;
2451   }
2452 };
2453
2454 int RGWSetRequestPayment_ObjStore_S3::get_params()
2455 {
2456   char *data;
2457   int len = 0;
2458   const auto max_size = s->cct->_conf->rgw_max_put_param_size;
2459   int r = rgw_rest_read_all_input(s, &data, &len, max_size, false);
2460
2461   if (r < 0) {
2462     return r;
2463   }
2464
2465   RGWSetRequestPaymentParser parser;
2466
2467   if (!parser.init()) {
2468     ldout(s->cct, 0) << "ERROR: failed to initialize parser" << dendl;
2469     r = -EIO;
2470     goto done;
2471   }
2472
2473   if (!parser.parse(data, len, 1)) {
2474     ldout(s->cct, 10) << "failed to parse data: " << data << dendl;
2475     r = -EINVAL;
2476     goto done;
2477   }
2478
2479   r = parser.get_request_payment_payer(&requester_pays);
2480
2481 done:
2482   free(data);
2483
2484   return r;
2485 }
2486
2487 void RGWSetRequestPayment_ObjStore_S3::send_response()
2488 {
2489   if (op_ret)
2490     set_req_state_err(s, op_ret);
2491   dump_errno(s);
2492   end_header(s);
2493 }
2494
2495 int RGWInitMultipart_ObjStore_S3::get_params()
2496 {
2497   RGWAccessControlPolicy_S3 s3policy(s->cct);
2498   op_ret = create_s3_policy(s, store, s3policy, s->owner);
2499   if (op_ret < 0)
2500     return op_ret;
2501
2502   policy = s3policy;
2503
2504   return 0;
2505 }
2506
2507 void RGWInitMultipart_ObjStore_S3::send_response()
2508 {
2509   if (op_ret)
2510     set_req_state_err(s, op_ret);
2511   dump_errno(s);
2512   for (auto &it : crypt_http_responses)
2513      dump_header(s, it.first, it.second);
2514   end_header(s, this, "application/xml");
2515   if (op_ret == 0) {
2516     dump_start(s);
2517     s->formatter->open_object_section_in_ns("InitiateMultipartUploadResult", XMLNS_AWS_S3);
2518     if (!s->bucket_tenant.empty())
2519       s->formatter->dump_string("Tenant", s->bucket_tenant);
2520     s->formatter->dump_string("Bucket", s->bucket_name);
2521     s->formatter->dump_string("Key", s->object.name);
2522     s->formatter->dump_string("UploadId", upload_id);
2523     s->formatter->close_section();
2524     rgw_flush_formatter_and_reset(s, s->formatter);
2525   }
2526 }
2527
2528 int RGWInitMultipart_ObjStore_S3::prepare_encryption(map<string, bufferlist>& attrs)
2529 {
2530   int res = 0;
2531   res = rgw_s3_prepare_encrypt(s, attrs, nullptr, nullptr, crypt_http_responses);
2532   return res;
2533 }
2534
2535 int RGWCompleteMultipart_ObjStore_S3::get_params()
2536 {
2537   int ret = RGWCompleteMultipart_ObjStore::get_params();
2538   if (ret < 0) {
2539     return ret;
2540   }
2541
2542   return do_aws4_auth_completion();
2543 }
2544
2545 void RGWCompleteMultipart_ObjStore_S3::send_response()
2546 {
2547   if (op_ret)
2548     set_req_state_err(s, op_ret);
2549   dump_errno(s);
2550   end_header(s, this, "application/xml");
2551   if (op_ret == 0) {
2552     dump_start(s);
2553     s->formatter->open_object_section_in_ns("CompleteMultipartUploadResult", XMLNS_AWS_S3);
2554     std::string base_uri = compute_domain_uri(s);
2555     if (!s->bucket_tenant.empty()) {
2556       s->formatter->dump_format("Location", "%s/%s:%s/%s",
2557           base_uri.c_str(),
2558           s->bucket_tenant.c_str(),
2559           s->bucket_name.c_str(),
2560           s->object.name.c_str()
2561           );
2562       s->formatter->dump_string("Tenant", s->bucket_tenant);
2563     } else {
2564       s->formatter->dump_format("Location", "%s/%s/%s",
2565           base_uri.c_str(),
2566           s->bucket_name.c_str(),
2567           s->object.name.c_str()
2568           );
2569     }
2570     s->formatter->dump_string("Bucket", s->bucket_name);
2571     s->formatter->dump_string("Key", s->object.name);
2572     s->formatter->dump_string("ETag", etag);
2573     s->formatter->close_section();
2574     rgw_flush_formatter_and_reset(s, s->formatter);
2575   }
2576 }
2577
2578 void RGWAbortMultipart_ObjStore_S3::send_response()
2579 {
2580   int r = op_ret;
2581   if (!r)
2582     r = STATUS_NO_CONTENT;
2583
2584   set_req_state_err(s, r);
2585   dump_errno(s);
2586   end_header(s, this);
2587 }
2588
2589 void RGWListMultipart_ObjStore_S3::send_response()
2590 {
2591   if (op_ret)
2592     set_req_state_err(s, op_ret);
2593   dump_errno(s);
2594   end_header(s, this, "application/xml");
2595
2596   if (op_ret == 0) {
2597     dump_start(s);
2598     s->formatter->open_object_section_in_ns("ListPartsResult", XMLNS_AWS_S3);
2599     map<uint32_t, RGWUploadPartInfo>::iterator iter;
2600     map<uint32_t, RGWUploadPartInfo>::reverse_iterator test_iter;
2601     int cur_max = 0;
2602
2603     iter = parts.begin();
2604     test_iter = parts.rbegin();
2605     if (test_iter != parts.rend()) {
2606       cur_max = test_iter->first;
2607     }
2608     if (!s->bucket_tenant.empty())
2609       s->formatter->dump_string("Tenant", s->bucket_tenant);
2610     s->formatter->dump_string("Bucket", s->bucket_name);
2611     s->formatter->dump_string("Key", s->object.name);
2612     s->formatter->dump_string("UploadId", upload_id);
2613     s->formatter->dump_string("StorageClass", "STANDARD");
2614     s->formatter->dump_int("PartNumberMarker", marker);
2615     s->formatter->dump_int("NextPartNumberMarker", cur_max);
2616     s->formatter->dump_int("MaxParts", max_parts);
2617     s->formatter->dump_string("IsTruncated", (truncated ? "true" : "false"));
2618
2619     ACLOwner& owner = policy.get_owner();
2620     dump_owner(s, owner.get_id(), owner.get_display_name());
2621
2622     for (; iter != parts.end(); ++iter) {
2623       RGWUploadPartInfo& info = iter->second;
2624
2625       s->formatter->open_object_section("Part");
2626
2627       dump_time(s, "LastModified", &info.modified);
2628
2629       s->formatter->dump_unsigned("PartNumber", info.num);
2630       s->formatter->dump_format("ETag", "\"%s\"", info.etag.c_str());
2631       s->formatter->dump_unsigned("Size", info.accounted_size);
2632       s->formatter->close_section();
2633     }
2634     s->formatter->close_section();
2635     rgw_flush_formatter_and_reset(s, s->formatter);
2636   }
2637 }
2638
2639 void RGWListBucketMultiparts_ObjStore_S3::send_response()
2640 {
2641   if (op_ret < 0)
2642     set_req_state_err(s, op_ret);
2643   dump_errno(s);
2644
2645   end_header(s, this, "application/xml");
2646   dump_start(s);
2647   if (op_ret < 0)
2648     return;
2649
2650   s->formatter->open_object_section_in_ns("ListMultipartUploadsResult", XMLNS_AWS_S3);
2651   if (!s->bucket_tenant.empty())
2652     s->formatter->dump_string("Tenant", s->bucket_tenant);
2653   s->formatter->dump_string("Bucket", s->bucket_name);
2654   if (!prefix.empty())
2655     s->formatter->dump_string("ListMultipartUploadsResult.Prefix", prefix);
2656   string& key_marker = marker.get_key();
2657   if (!key_marker.empty())
2658     s->formatter->dump_string("KeyMarker", key_marker);
2659   string& upload_id_marker = marker.get_upload_id();
2660   if (!upload_id_marker.empty())
2661     s->formatter->dump_string("UploadIdMarker", upload_id_marker);
2662   string next_key = next_marker.mp.get_key();
2663   if (!next_key.empty())
2664     s->formatter->dump_string("NextKeyMarker", next_key);
2665   string next_upload_id = next_marker.mp.get_upload_id();
2666   if (!next_upload_id.empty())
2667     s->formatter->dump_string("NextUploadIdMarker", next_upload_id);
2668   s->formatter->dump_int("MaxUploads", max_uploads);
2669   if (!delimiter.empty())
2670     s->formatter->dump_string("Delimiter", delimiter);
2671   s->formatter->dump_string("IsTruncated", (is_truncated ? "true" : "false"));
2672
2673   if (op_ret >= 0) {
2674     vector<RGWMultipartUploadEntry>::iterator iter;
2675     for (iter = uploads.begin(); iter != uploads.end(); ++iter) {
2676       RGWMPObj& mp = iter->mp;
2677       s->formatter->open_array_section("Upload");
2678       s->formatter->dump_string("Key", mp.get_key());
2679       s->formatter->dump_string("UploadId", mp.get_upload_id());
2680       dump_owner(s, s->user->user_id, s->user->display_name, "Initiator");
2681       dump_owner(s, s->user->user_id, s->user->display_name);
2682       s->formatter->dump_string("StorageClass", "STANDARD");
2683       dump_time(s, "Initiated", &iter->obj.meta.mtime);
2684       s->formatter->close_section();
2685     }
2686     if (!common_prefixes.empty()) {
2687       s->formatter->open_array_section("CommonPrefixes");
2688       map<string, bool>::iterator pref_iter;
2689       for (pref_iter = common_prefixes.begin();
2690            pref_iter != common_prefixes.end(); ++pref_iter) {
2691         s->formatter->dump_string("CommonPrefixes.Prefix", pref_iter->first);
2692       }
2693       s->formatter->close_section();
2694     }
2695   }
2696   s->formatter->close_section();
2697   rgw_flush_formatter_and_reset(s, s->formatter);
2698 }
2699
2700 int RGWDeleteMultiObj_ObjStore_S3::get_params()
2701 {
2702   int ret = RGWDeleteMultiObj_ObjStore::get_params();
2703   if (ret < 0) {
2704     return ret;
2705   }
2706
2707   return do_aws4_auth_completion();
2708 }
2709
2710 void RGWDeleteMultiObj_ObjStore_S3::send_status()
2711 {
2712   if (! status_dumped) {
2713     if (op_ret < 0)
2714       set_req_state_err(s, op_ret);
2715     dump_errno(s);
2716     status_dumped = true;
2717   }
2718 }
2719
2720 void RGWDeleteMultiObj_ObjStore_S3::begin_response()
2721 {
2722
2723   if (!status_dumped) {
2724     send_status();
2725   }
2726
2727   dump_start(s);
2728   end_header(s, this, "application/xml");
2729   s->formatter->open_object_section_in_ns("DeleteResult", XMLNS_AWS_S3);
2730
2731   rgw_flush_formatter(s, s->formatter);
2732 }
2733
2734 void RGWDeleteMultiObj_ObjStore_S3::send_partial_response(rgw_obj_key& key,
2735                                                           bool delete_marker,
2736                                                           const string& marker_version_id, int ret)
2737 {
2738   if (!key.empty()) {
2739     if (op_ret == 0 && !quiet) {
2740       s->formatter->open_object_section("Deleted");
2741       s->formatter->dump_string("Key", key.name);
2742       if (!key.instance.empty()) {
2743         s->formatter->dump_string("VersionId", key.instance);
2744       }
2745       if (delete_marker) {
2746         s->formatter->dump_bool("DeleteMarker", true);
2747         s->formatter->dump_string("DeleteMarkerVersionId", marker_version_id);
2748       }
2749       s->formatter->close_section();
2750     } else if (op_ret < 0) {
2751       struct rgw_http_error r;
2752       int err_no;
2753
2754       s->formatter->open_object_section("Error");
2755
2756       err_no = -op_ret;
2757       rgw_get_errno_s3(&r, err_no);
2758
2759       s->formatter->dump_string("Key", key.name);
2760       s->formatter->dump_string("VersionId", key.instance);
2761       s->formatter->dump_int("Code", r.http_ret);
2762       s->formatter->dump_string("Message", r.s3_code);
2763       s->formatter->close_section();
2764     }
2765
2766     rgw_flush_formatter(s, s->formatter);
2767   }
2768 }
2769
2770 void RGWDeleteMultiObj_ObjStore_S3::end_response()
2771 {
2772
2773   s->formatter->close_section();
2774   rgw_flush_formatter_and_reset(s, s->formatter);
2775 }
2776
2777 void RGWGetObjLayout_ObjStore_S3::send_response()
2778 {
2779   if (op_ret)
2780     set_req_state_err(s, op_ret);
2781   dump_errno(s);
2782   end_header(s, this, "application/json");
2783
2784   JSONFormatter f;
2785
2786   if (op_ret < 0) {
2787     return;
2788   }
2789
2790   f.open_object_section("result");
2791   ::encode_json("head", head_obj, &f);
2792   ::encode_json("manifest", *manifest, &f);
2793   f.open_array_section("data_location");
2794   for (auto miter = manifest->obj_begin(); miter != manifest->obj_end(); ++miter) {
2795     f.open_object_section("obj");
2796     rgw_raw_obj raw_loc = miter.get_location().get_raw_obj(store);
2797     ::encode_json("ofs", miter.get_ofs(), &f);
2798     ::encode_json("loc", raw_loc, &f);
2799     ::encode_json("loc_ofs", miter.location_ofs(), &f);
2800     ::encode_json("loc_size", miter.get_stripe_size(), &f);
2801     f.close_section();
2802     rgw_flush_formatter(s, &f);
2803   }
2804   f.close_section();
2805   f.close_section();
2806   rgw_flush_formatter(s, &f);
2807 }
2808
2809 int RGWConfigBucketMetaSearch_ObjStore_S3::get_params()
2810 {
2811   auto iter = s->info.x_meta_map.find("x-amz-meta-search");
2812   if (iter == s->info.x_meta_map.end()) {
2813     s->err.message = "X-Rgw-Meta-Search header not provided";
2814     ldout(s->cct, 5) << s->err.message << dendl;
2815     return -EINVAL;
2816   }
2817
2818   list<string> expressions;
2819   get_str_list(iter->second, ",", expressions);
2820
2821   for (auto& expression : expressions) {
2822     vector<string> args;
2823     get_str_vec(expression, ";", args);
2824
2825     if (args.empty()) {
2826       s->err.message = "invalid empty expression";
2827       ldout(s->cct, 5) << s->err.message << dendl;
2828       return -EINVAL;
2829     }
2830     if (args.size() > 2) {
2831       s->err.message = string("invalid expression: ") + expression;
2832       ldout(s->cct, 5) << s->err.message << dendl;
2833       return -EINVAL;
2834     }
2835
2836     string key = boost::algorithm::to_lower_copy(rgw_trim_whitespace(args[0]));
2837     string val;
2838     if (args.size() > 1) {
2839       val = boost::algorithm::to_lower_copy(rgw_trim_whitespace(args[1]));
2840     }
2841
2842     if (!boost::algorithm::starts_with(key, RGW_AMZ_META_PREFIX)) {
2843       s->err.message = string("invalid expression, key must start with '" RGW_AMZ_META_PREFIX "' : ") + expression;
2844       ldout(s->cct, 5) << s->err.message << dendl;
2845       return -EINVAL;
2846     }
2847
2848     key = key.substr(sizeof(RGW_AMZ_META_PREFIX) - 1);
2849
2850     ESEntityTypeMap::EntityType entity_type;
2851
2852     if (val.empty() || val == "str" || val == "string") {
2853       entity_type = ESEntityTypeMap::ES_ENTITY_STR;
2854     } else if (val == "int" || val == "integer") {
2855       entity_type = ESEntityTypeMap::ES_ENTITY_INT;
2856     } else if (val == "date" || val == "datetime") {
2857       entity_type = ESEntityTypeMap::ES_ENTITY_DATE;
2858     } else {
2859       s->err.message = string("invalid entity type: ") + val;
2860       ldout(s->cct, 5) << s->err.message << dendl;
2861       return -EINVAL;
2862     }
2863
2864     mdsearch_config[key] = entity_type;
2865   }
2866
2867   return 0;
2868 }
2869
2870 void RGWConfigBucketMetaSearch_ObjStore_S3::send_response()
2871 {
2872   if (op_ret)
2873     set_req_state_err(s, op_ret);
2874   dump_errno(s);
2875   end_header(s, this);
2876 }
2877
2878 void RGWGetBucketMetaSearch_ObjStore_S3::send_response()
2879 {
2880   if (op_ret)
2881     set_req_state_err(s, op_ret);
2882   dump_errno(s);
2883   end_header(s, NULL, "application/xml");
2884
2885   Formatter *f = s->formatter;
2886   f->open_array_section("GetBucketMetaSearchResult");
2887   for (auto& e : s->bucket_info.mdsearch_config) {
2888     f->open_object_section("Entry");
2889     string k = string("x-amz-meta-") + e.first;
2890     f->dump_string("Key", k.c_str());
2891     const char *type;
2892     switch (e.second) {
2893       case ESEntityTypeMap::ES_ENTITY_INT:
2894         type = "int";
2895         break;
2896       case ESEntityTypeMap::ES_ENTITY_DATE:
2897         type = "date";
2898         break;
2899       default:
2900         type = "str";
2901     }
2902     f->dump_string("Type", type);
2903     f->close_section();
2904   }
2905   f->close_section();
2906   rgw_flush_formatter(s, f);
2907 }
2908
2909 void RGWDelBucketMetaSearch_ObjStore_S3::send_response()
2910 {
2911   if (op_ret)
2912     set_req_state_err(s, op_ret);
2913   dump_errno(s);
2914   end_header(s, this);
2915 }
2916
2917
2918 RGWOp *RGWHandler_REST_Service_S3::op_get()
2919 {
2920   if (is_usage_op()) {
2921     return new RGWGetUsage_ObjStore_S3;
2922   } else {
2923     return new RGWListBuckets_ObjStore_S3;
2924   }
2925 }
2926
2927 RGWOp *RGWHandler_REST_Service_S3::op_head()
2928 {
2929   return new RGWListBuckets_ObjStore_S3;
2930 }
2931
2932 RGWOp *RGWHandler_REST_Service_S3::op_post()
2933 {
2934   if (s->info.args.exists("Action")) {
2935     string action = s->info.args.get("Action");
2936     if (action.compare("CreateRole") == 0)
2937       return new RGWCreateRole;
2938     if (action.compare("DeleteRole") == 0)
2939       return new RGWDeleteRole;
2940     if (action.compare("GetRole") == 0)
2941       return new RGWGetRole;
2942     if (action.compare("UpdateAssumeRolePolicy") == 0)
2943       return new RGWModifyRole;
2944     if (action.compare("ListRoles") == 0)
2945       return new RGWListRoles;
2946     if (action.compare("PutRolePolicy") == 0)
2947       return new RGWPutRolePolicy;
2948     if (action.compare("GetRolePolicy") == 0)
2949       return new RGWGetRolePolicy;
2950     if (action.compare("ListRolePolicies") == 0)
2951       return new RGWListRolePolicies;
2952     if (action.compare("DeleteRolePolicy") == 0)
2953       return new RGWDeleteRolePolicy;
2954   }
2955   return NULL;
2956 }
2957
2958 RGWOp *RGWHandler_REST_Bucket_S3::get_obj_op(bool get_data)
2959 {
2960   // Non-website mode
2961   if (get_data) {
2962     return new RGWListBucket_ObjStore_S3;
2963   } else {
2964     return new RGWStatBucket_ObjStore_S3;
2965   }
2966 }
2967
2968 RGWOp *RGWHandler_REST_Bucket_S3::op_get()
2969 {
2970   if (s->info.args.sub_resource_exists("logging"))
2971     return new RGWGetBucketLogging_ObjStore_S3;
2972
2973   if (s->info.args.sub_resource_exists("location"))
2974     return new RGWGetBucketLocation_ObjStore_S3;
2975
2976   if (s->info.args.sub_resource_exists("versioning"))
2977     return new RGWGetBucketVersioning_ObjStore_S3;
2978
2979   if (s->info.args.sub_resource_exists("website")) {
2980     if (!s->cct->_conf->rgw_enable_static_website) {
2981       return NULL;
2982     }
2983     return new RGWGetBucketWebsite_ObjStore_S3;
2984   }
2985
2986   if (s->info.args.exists("mdsearch")) {
2987     return new RGWGetBucketMetaSearch_ObjStore_S3;
2988   }
2989
2990   if (is_acl_op()) {
2991     return new RGWGetACLs_ObjStore_S3;
2992   } else if (is_cors_op()) {
2993     return new RGWGetCORS_ObjStore_S3;
2994   } else if (is_request_payment_op()) {
2995     return new RGWGetRequestPayment_ObjStore_S3;
2996   } else if (s->info.args.exists("uploads")) {
2997     return new RGWListBucketMultiparts_ObjStore_S3;
2998   } else if(is_lc_op()) {
2999     return new RGWGetLC_ObjStore_S3;
3000   } else if(is_policy_op()) {
3001     return new RGWGetBucketPolicy;
3002   }
3003   return get_obj_op(true);
3004 }
3005
3006 RGWOp *RGWHandler_REST_Bucket_S3::op_head()
3007 {
3008   if (is_acl_op()) {
3009     return new RGWGetACLs_ObjStore_S3;
3010   } else if (s->info.args.exists("uploads")) {
3011     return new RGWListBucketMultiparts_ObjStore_S3;
3012   }
3013   return get_obj_op(false);
3014 }
3015
3016 RGWOp *RGWHandler_REST_Bucket_S3::op_put()
3017 {
3018   if (s->info.args.sub_resource_exists("logging"))
3019     return NULL;
3020   if (s->info.args.sub_resource_exists("versioning"))
3021     return new RGWSetBucketVersioning_ObjStore_S3;
3022   if (s->info.args.sub_resource_exists("website")) {
3023     if (!s->cct->_conf->rgw_enable_static_website) {
3024       return NULL;
3025     }
3026     return new RGWSetBucketWebsite_ObjStore_S3;
3027   }
3028   if (is_acl_op()) {
3029     return new RGWPutACLs_ObjStore_S3;
3030   } else if (is_cors_op()) {
3031     return new RGWPutCORS_ObjStore_S3;
3032   } else if (is_request_payment_op()) {
3033     return new RGWSetRequestPayment_ObjStore_S3;
3034   } else if(is_lc_op()) {
3035     return new RGWPutLC_ObjStore_S3;
3036   } else if(is_policy_op()) {
3037     return new RGWPutBucketPolicy;
3038   }
3039   return new RGWCreateBucket_ObjStore_S3;
3040 }
3041
3042 RGWOp *RGWHandler_REST_Bucket_S3::op_delete()
3043 {
3044   if (is_cors_op()) {
3045     return new RGWDeleteCORS_ObjStore_S3;
3046   } else if(is_lc_op()) {
3047     return new RGWDeleteLC_ObjStore_S3;
3048   } else if(is_policy_op()) {
3049     return new RGWDeleteBucketPolicy;
3050   }
3051
3052   if (s->info.args.sub_resource_exists("website")) {
3053     if (!s->cct->_conf->rgw_enable_static_website) {
3054       return NULL;
3055     }
3056     return new RGWDeleteBucketWebsite_ObjStore_S3;
3057   }
3058
3059   if (s->info.args.exists("mdsearch")) {
3060     return new RGWDelBucketMetaSearch_ObjStore_S3;
3061   }
3062
3063   return new RGWDeleteBucket_ObjStore_S3;
3064 }
3065
3066 RGWOp *RGWHandler_REST_Bucket_S3::op_post()
3067 {
3068   if (s->info.args.exists("delete")) {
3069     return new RGWDeleteMultiObj_ObjStore_S3;
3070   }
3071
3072   if (s->info.args.exists("mdsearch")) {
3073     return new RGWConfigBucketMetaSearch_ObjStore_S3;
3074   }
3075
3076   return new RGWPostObj_ObjStore_S3;
3077 }
3078
3079 RGWOp *RGWHandler_REST_Bucket_S3::op_options()
3080 {
3081   return new RGWOptionsCORS_ObjStore_S3;
3082 }
3083
3084 RGWOp *RGWHandler_REST_Obj_S3::get_obj_op(bool get_data)
3085 {
3086   if (is_acl_op()) {
3087     return new RGWGetACLs_ObjStore_S3;
3088   }
3089   RGWGetObj_ObjStore_S3 *get_obj_op = new RGWGetObj_ObjStore_S3;
3090   get_obj_op->set_get_data(get_data);
3091   return get_obj_op;
3092 }
3093
3094 RGWOp *RGWHandler_REST_Obj_S3::op_get()
3095 {
3096   if (is_acl_op()) {
3097     return new RGWGetACLs_ObjStore_S3;
3098   } else if (s->info.args.exists("uploadId")) {
3099     return new RGWListMultipart_ObjStore_S3;
3100   } else if (s->info.args.exists("layout")) {
3101     return new RGWGetObjLayout_ObjStore_S3;
3102   } else if (is_tagging_op()) {
3103     return new RGWGetObjTags_ObjStore_S3;
3104   }
3105   return get_obj_op(true);
3106 }
3107
3108 RGWOp *RGWHandler_REST_Obj_S3::op_head()
3109 {
3110   if (is_acl_op()) {
3111     return new RGWGetACLs_ObjStore_S3;
3112   } else if (s->info.args.exists("uploadId")) {
3113     return new RGWListMultipart_ObjStore_S3;
3114   }
3115   return get_obj_op(false);
3116 }
3117
3118 RGWOp *RGWHandler_REST_Obj_S3::op_put()
3119 {
3120   if (is_acl_op()) {
3121     return new RGWPutACLs_ObjStore_S3;
3122   } else if (is_tagging_op()) {
3123     return new RGWPutObjTags_ObjStore_S3;
3124   }
3125
3126   if (s->init_state.src_bucket.empty())
3127     return new RGWPutObj_ObjStore_S3;
3128   else
3129     return new RGWCopyObj_ObjStore_S3;
3130 }
3131
3132 RGWOp *RGWHandler_REST_Obj_S3::op_delete()
3133 {
3134   if (is_tagging_op()) {
3135     return new RGWDeleteObjTags_ObjStore_S3;
3136   }
3137   string upload_id = s->info.args.get("uploadId");
3138
3139   if (upload_id.empty())
3140     return new RGWDeleteObj_ObjStore_S3;
3141   else
3142     return new RGWAbortMultipart_ObjStore_S3;
3143 }
3144
3145 RGWOp *RGWHandler_REST_Obj_S3::op_post()
3146 {
3147   if (s->info.args.exists("uploadId"))
3148     return new RGWCompleteMultipart_ObjStore_S3;
3149
3150   if (s->info.args.exists("uploads"))
3151     return new RGWInitMultipart_ObjStore_S3;
3152
3153   return new RGWPostObj_ObjStore_S3;
3154 }
3155
3156 RGWOp *RGWHandler_REST_Obj_S3::op_options()
3157 {
3158   return new RGWOptionsCORS_ObjStore_S3;
3159 }
3160
3161 int RGWHandler_REST_S3::init_from_header(struct req_state* s,
3162                                         int default_formatter,
3163                                         bool configurable_format)
3164 {
3165   string req;
3166   string first;
3167
3168   const char *req_name = s->relative_uri.c_str();
3169   const char *p;
3170
3171   if (*req_name == '?') {
3172     p = req_name;
3173   } else {
3174     p = s->info.request_params.c_str();
3175   }
3176
3177   s->info.args.set(p);
3178   s->info.args.parse();
3179
3180   /* must be called after the args parsing */
3181   int ret = allocate_formatter(s, default_formatter, configurable_format);
3182   if (ret < 0)
3183     return ret;
3184
3185   if (*req_name != '/')
3186     return 0;
3187
3188   req_name++;
3189
3190   if (!*req_name)
3191     return 0;
3192
3193   req = req_name;
3194   int pos = req.find('/');
3195   if (pos >= 0) {
3196     first = req.substr(0, pos);
3197   } else {
3198     first = req;
3199   }
3200
3201   /*
3202    * XXX The intent of the check for empty is apparently to let the bucket
3203    * name from DNS to be set ahead. However, we currently take the DNS
3204    * bucket and re-insert it into URL in rgw_rest.cc:RGWREST::preprocess().
3205    * So, this check is meaningless.
3206    *
3207    * Rather than dropping this, the code needs to be changed into putting
3208    * the bucket (and its tenant) from DNS and Host: header (HTTP_HOST)
3209    * into req_status.bucket_name directly.
3210    */
3211   if (s->init_state.url_bucket.empty()) {
3212     // Save bucket to tide us over until token is parsed.
3213     s->init_state.url_bucket = first;
3214     if (pos >= 0) {
3215       string encoded_obj_str = req.substr(pos+1);
3216       s->object = rgw_obj_key(encoded_obj_str, s->info.args.get("versionId"));
3217     }
3218   } else {
3219     s->object = rgw_obj_key(req_name, s->info.args.get("versionId"));
3220   }
3221   return 0;
3222 }
3223
3224 int RGWHandler_REST_S3::postauth_init()
3225 {
3226   struct req_init_state *t = &s->init_state;
3227   bool relaxed_names = s->cct->_conf->rgw_relaxed_s3_bucket_names;
3228
3229   rgw_parse_url_bucket(t->url_bucket, s->user->user_id.tenant,
3230                       s->bucket_tenant, s->bucket_name);
3231
3232   dout(10) << "s->object=" << (!s->object.empty() ? s->object : rgw_obj_key("<NULL>"))
3233            << " s->bucket=" << rgw_make_bucket_entry_name(s->bucket_tenant, s->bucket_name) << dendl;
3234
3235   int ret;
3236   ret = rgw_validate_tenant_name(s->bucket_tenant);
3237   if (ret)
3238     return ret;
3239   if (!s->bucket_name.empty()) {
3240     ret = valid_s3_bucket_name(s->bucket_name, relaxed_names);
3241     if (ret)
3242       return ret;
3243     ret = validate_object_name(s->object.name);
3244     if (ret)
3245       return ret;
3246   }
3247
3248   if (!t->src_bucket.empty()) {
3249     rgw_parse_url_bucket(t->src_bucket, s->user->user_id.tenant,
3250                         s->src_tenant_name, s->src_bucket_name);
3251     ret = rgw_validate_tenant_name(s->src_tenant_name);
3252     if (ret)
3253       return ret;
3254     ret = valid_s3_bucket_name(s->src_bucket_name, relaxed_names);
3255     if (ret)
3256       return ret;
3257   }
3258   return 0;
3259 }
3260
3261 int RGWHandler_REST_S3::init(RGWRados *store, struct req_state *s,
3262                              rgw::io::BasicClient *cio)
3263 {
3264   int ret;
3265
3266   s->dialect = "s3";
3267
3268   ret = rgw_validate_tenant_name(s->bucket_tenant);
3269   if (ret)
3270     return ret;
3271   bool relaxed_names = s->cct->_conf->rgw_relaxed_s3_bucket_names;
3272   if (!s->bucket_name.empty()) {
3273     ret = valid_s3_bucket_name(s->bucket_name, relaxed_names);
3274     if (ret)
3275       return ret;
3276     ret = validate_object_name(s->object.name);
3277     if (ret)
3278       return ret;
3279   }
3280
3281   const char *cacl = s->info.env->get("HTTP_X_AMZ_ACL");
3282   if (cacl)
3283     s->canned_acl = cacl;
3284
3285   s->has_acl_header = s->info.env->exists_prefix("HTTP_X_AMZ_GRANT");
3286
3287   const char *copy_source = s->info.env->get("HTTP_X_AMZ_COPY_SOURCE");
3288   if (copy_source &&
3289       (! s->info.env->get("HTTP_X_AMZ_COPY_SOURCE_RANGE")) &&
3290       (! s->info.args.exists("uploadId"))) {
3291
3292     ret = RGWCopyObj::parse_copy_location(url_decode(copy_source),
3293                                           s->init_state.src_bucket,
3294                                           s->src_object);
3295     if (!ret) {
3296       ldout(s->cct, 0) << "failed to parse copy location" << dendl;
3297       return -EINVAL; // XXX why not -ERR_INVALID_BUCKET_NAME or -ERR_BAD_URL?
3298     }
3299   }
3300
3301   return RGWHandler_REST::init(store, s, cio);
3302 }
3303
3304 enum class AwsVersion {
3305   UNKOWN,
3306   V2,
3307   V4
3308 };
3309
3310 enum class AwsRoute {
3311   UNKOWN,
3312   QUERY_STRING,
3313   HEADERS
3314 };
3315
3316 static inline std::pair<AwsVersion, AwsRoute>
3317 discover_aws_flavour(const req_info& info)
3318 {
3319   using rgw::auth::s3::AWS4_HMAC_SHA256_STR;
3320
3321   AwsVersion version = AwsVersion::UNKOWN;
3322   AwsRoute route = AwsRoute::UNKOWN;
3323
3324   const char* http_auth = info.env->get("HTTP_AUTHORIZATION");
3325   if (http_auth && http_auth[0]) {
3326     /* Authorization in Header */
3327     route = AwsRoute::HEADERS;
3328
3329     if (!strncmp(http_auth, AWS4_HMAC_SHA256_STR,
3330                  strlen(AWS4_HMAC_SHA256_STR))) {
3331       /* AWS v4 */
3332       version = AwsVersion::V4;
3333     } else if (!strncmp(http_auth, "AWS ", 4)) {
3334       /* AWS v2 */
3335       version = AwsVersion::V2;
3336     }
3337   } else {
3338     route = AwsRoute::QUERY_STRING;
3339
3340     if (info.args.get("X-Amz-Algorithm") == AWS4_HMAC_SHA256_STR) {
3341       /* AWS v4 */
3342       version = AwsVersion::V4;
3343     } else if (!info.args.get("AWSAccessKeyId").empty()) {
3344       /* AWS v2 */
3345       version = AwsVersion::V2;
3346     }
3347   }
3348
3349   return std::make_pair(version, route);
3350 }
3351
3352 static void init_anon_user(struct req_state *s)
3353 {
3354   rgw_get_anon_user(*(s->user));
3355   s->perm_mask = RGW_PERM_FULL_CONTROL;
3356 }
3357
3358 /*
3359  * verify that a signed request comes from the keyholder
3360  * by checking the signature against our locally-computed version
3361  *
3362  * it tries AWS v4 before AWS v2
3363  */
3364 int RGW_Auth_S3::authorize(RGWRados* const store,
3365                            const rgw::auth::StrategyRegistry& auth_registry,
3366                            struct req_state* const s)
3367 {
3368
3369   /* neither keystone and rados enabled; warn and exit! */
3370   if (!store->ctx()->_conf->rgw_s3_auth_use_rados &&
3371       !store->ctx()->_conf->rgw_s3_auth_use_keystone &&
3372       !store->ctx()->_conf->rgw_s3_auth_use_ldap) {
3373     dout(0) << "WARNING: no authorization backend enabled! Users will never authenticate." << dendl;
3374     return -EPERM;
3375   }
3376
3377   const auto ret = rgw::auth::Strategy::apply(auth_registry.get_s3_main(), s);
3378   if (ret == 0) {
3379     /* Populate the owner info. */
3380     s->owner.set_id(s->user->user_id);
3381     s->owner.set_name(s->user->display_name);
3382   }
3383   return ret;
3384 }
3385
3386 int RGWHandler_Auth_S3::init(RGWRados *store, struct req_state *state,
3387                              rgw::io::BasicClient *cio)
3388 {
3389   int ret = RGWHandler_REST_S3::init_from_header(state, RGW_FORMAT_JSON,
3390                                                      true);
3391   if (ret < 0)
3392     return ret;
3393
3394   return RGWHandler_REST::init(store, state, cio);
3395 }
3396
3397 RGWHandler_REST* RGWRESTMgr_S3::get_handler(struct req_state* const s,
3398                                             const rgw::auth::StrategyRegistry& auth_registry,
3399                                             const std::string& frontend_prefix)
3400 {
3401   bool is_s3website = enable_s3website && (s->prot_flags & RGW_REST_WEBSITE);
3402   int ret =
3403     RGWHandler_REST_S3::init_from_header(s,
3404                                         is_s3website ? RGW_FORMAT_HTML :
3405                                         RGW_FORMAT_XML, true);
3406   if (ret < 0)
3407     return NULL;
3408
3409   RGWHandler_REST* handler;
3410   // TODO: Make this more readable
3411   if (is_s3website) {
3412     if (s->init_state.url_bucket.empty()) {
3413       handler = new RGWHandler_REST_Service_S3Website(auth_registry);
3414     } else if (s->object.empty()) {
3415       handler = new RGWHandler_REST_Bucket_S3Website(auth_registry);
3416     } else {
3417       handler = new RGWHandler_REST_Obj_S3Website(auth_registry);
3418     }
3419   } else {
3420     if (s->init_state.url_bucket.empty()) {
3421       handler = new RGWHandler_REST_Service_S3(auth_registry);
3422     } else if (s->object.empty()) {
3423       handler = new RGWHandler_REST_Bucket_S3(auth_registry);
3424     } else {
3425       handler = new RGWHandler_REST_Obj_S3(auth_registry);
3426     }
3427   }
3428
3429   ldout(s->cct, 20) << __func__ << " handler=" << typeid(*handler).name()
3430                     << dendl;
3431   return handler;
3432 }
3433
3434 bool RGWHandler_REST_S3Website::web_dir() const {
3435   std::string subdir_name = url_decode(s->object.name);
3436
3437   if (subdir_name.empty()) {
3438     return false;
3439   } else if (subdir_name.back() == '/') {
3440     subdir_name.pop_back();
3441   }
3442
3443   rgw_obj obj(s->bucket, subdir_name);
3444
3445   RGWObjectCtx& obj_ctx = *static_cast<RGWObjectCtx *>(s->obj_ctx);
3446   obj_ctx.obj.set_atomic(obj);
3447   obj_ctx.obj.set_prefetch_data(obj);
3448
3449   RGWObjState* state = nullptr;
3450   if (store->get_obj_state(&obj_ctx, s->bucket_info, obj, &state, false) < 0) {
3451     return false;
3452   }
3453   if (! state->exists) {
3454     return false;
3455   }
3456   return state->exists;
3457 }
3458
3459 int RGWHandler_REST_S3Website::init(RGWRados *store, req_state *s,
3460                                     rgw::io::BasicClient* cio)
3461 {
3462   // save the original object name before retarget() replaces it with the
3463   // result of get_effective_key(). the error_handler() needs the original
3464   // object name for redirect handling
3465   original_object_name = s->object.name;
3466
3467   return RGWHandler_REST_S3::init(store, s, cio);
3468 }
3469
3470 int RGWHandler_REST_S3Website::retarget(RGWOp* op, RGWOp** new_op) {
3471   *new_op = op;
3472   ldout(s->cct, 10) << __func__ << "Starting retarget" << dendl;
3473
3474   if (!(s->prot_flags & RGW_REST_WEBSITE))
3475     return 0;
3476
3477   RGWObjectCtx& obj_ctx = *static_cast<RGWObjectCtx *>(s->obj_ctx);
3478   int ret = store->get_bucket_info(obj_ctx, s->bucket_tenant,
3479                                   s->bucket_name, s->bucket_info, NULL,
3480                                   &s->bucket_attrs);
3481   if (ret < 0) {
3482       // TODO-FUTURE: if the bucket does not exist, maybe expose it here?
3483       return -ERR_NO_SUCH_BUCKET;
3484   }
3485   if (!s->bucket_info.has_website) {
3486       // TODO-FUTURE: if the bucket has no WebsiteConfig, expose it here
3487       return -ERR_NO_SUCH_WEBSITE_CONFIGURATION;
3488   }
3489
3490   rgw_obj_key new_obj;
3491   s->bucket_info.website_conf.get_effective_key(s->object.name, &new_obj.name, web_dir());
3492   ldout(s->cct, 10) << "retarget get_effective_key " << s->object << " -> "
3493                     << new_obj << dendl;
3494
3495   RGWBWRoutingRule rrule;
3496   bool should_redirect =
3497     s->bucket_info.website_conf.should_redirect(new_obj.name, 0, &rrule);
3498
3499   if (should_redirect) {
3500     const string& hostname = s->info.env->get("HTTP_HOST", "");
3501     const string& protocol =
3502       (s->info.env->get("SERVER_PORT_SECURE") ? "https" : "http");
3503     int redirect_code = 0;
3504     rrule.apply_rule(protocol, hostname, s->object.name, &s->redirect,
3505                     &redirect_code);
3506     // APply a custom HTTP response code
3507     if (redirect_code > 0)
3508       s->err.http_ret = redirect_code; // Apply a custom HTTP response code
3509     ldout(s->cct, 10) << "retarget redirect code=" << redirect_code
3510                       << " proto+host:" << protocol << "://" << hostname
3511                       << " -> " << s->redirect << dendl;
3512     return -ERR_WEBSITE_REDIRECT;
3513   }
3514
3515   /*
3516    * FIXME: if s->object != new_obj, drop op and create a new op to handle
3517    * operation. Or remove this comment if it's not applicable anymore
3518    */
3519
3520   s->object = new_obj;
3521
3522   return 0;
3523 }
3524
3525 RGWOp* RGWHandler_REST_S3Website::op_get()
3526 {
3527   return get_obj_op(true);
3528 }
3529
3530 RGWOp* RGWHandler_REST_S3Website::op_head()
3531 {
3532   return get_obj_op(false);
3533 }
3534
3535 int RGWHandler_REST_S3Website::serve_errordoc(int http_ret, const string& errordoc_key) {
3536   int ret = 0;
3537   s->formatter->reset(); /* Try to throw it all away */
3538
3539   std::shared_ptr<RGWGetObj_ObjStore_S3Website> getop( static_cast<RGWGetObj_ObjStore_S3Website*>(op_get()));
3540   if (getop.get() == NULL) {
3541     return -1; // Trigger double error handler
3542   }
3543   getop->init(store, s, this);
3544   getop->range_str = NULL;
3545   getop->if_mod = NULL;
3546   getop->if_unmod = NULL;
3547   getop->if_match = NULL;
3548   getop->if_nomatch = NULL;
3549   s->object = errordoc_key;
3550
3551   ret = init_permissions(getop.get());
3552   if (ret < 0) {
3553     ldout(s->cct, 20) << "serve_errordoc failed, init_permissions ret=" << ret << dendl;
3554     return -1; // Trigger double error handler
3555   }
3556
3557   ret = read_permissions(getop.get());
3558   if (ret < 0) {
3559     ldout(s->cct, 20) << "serve_errordoc failed, read_permissions ret=" << ret << dendl;
3560     return -1; // Trigger double error handler
3561   }
3562
3563   if (http_ret) {
3564      getop->set_custom_http_response(http_ret);
3565   }
3566
3567   ret = getop->init_processing();
3568   if (ret < 0) {
3569     ldout(s->cct, 20) << "serve_errordoc failed, init_processing ret=" << ret << dendl;
3570     return -1; // Trigger double error handler
3571   }
3572
3573   ret = getop->verify_op_mask();
3574   if (ret < 0) {
3575     ldout(s->cct, 20) << "serve_errordoc failed, verify_op_mask ret=" << ret << dendl;
3576     return -1; // Trigger double error handler
3577   }
3578
3579   ret = getop->verify_permission();
3580   if (ret < 0) {
3581     ldout(s->cct, 20) << "serve_errordoc failed, verify_permission ret=" << ret << dendl;
3582     return -1; // Trigger double error handler
3583   }
3584
3585   ret = getop->verify_params();
3586   if (ret < 0) {
3587     ldout(s->cct, 20) << "serve_errordoc failed, verify_params ret=" << ret << dendl;
3588     return -1; // Trigger double error handler
3589   }
3590
3591   // No going back now
3592   getop->pre_exec();
3593   /*
3594    * FIXME Missing headers:
3595    * With a working errordoc, the s3 error fields are rendered as HTTP headers,
3596    *   x-amz-error-code: NoSuchKey
3597    *   x-amz-error-message: The specified key does not exist.
3598    *   x-amz-error-detail-Key: foo
3599    */
3600   getop->execute();
3601   getop->complete();
3602   return 0;
3603
3604 }
3605
3606 int RGWHandler_REST_S3Website::error_handler(int err_no,
3607                                             string* error_content) {
3608   int new_err_no = -1;
3609   rgw_http_errors::const_iterator r = rgw_http_s3_errors.find(err_no > 0 ? err_no : -err_no);
3610   int http_error_code = -1;
3611
3612   if (r != rgw_http_s3_errors.end()) {
3613     http_error_code = r->second.first;
3614   }
3615   ldout(s->cct, 10) << "RGWHandler_REST_S3Website::error_handler err_no=" << err_no << " http_ret=" << http_error_code << dendl;
3616
3617   RGWBWRoutingRule rrule;
3618   bool should_redirect =
3619     s->bucket_info.website_conf.should_redirect(original_object_name,
3620                                                 http_error_code, &rrule);
3621
3622   if (should_redirect) {
3623     const string& hostname = s->info.env->get("HTTP_HOST", "");
3624     const string& protocol =
3625       (s->info.env->get("SERVER_PORT_SECURE") ? "https" : "http");
3626     int redirect_code = 0;
3627     rrule.apply_rule(protocol, hostname, original_object_name,
3628                      &s->redirect, &redirect_code);
3629     // Apply a custom HTTP response code
3630     if (redirect_code > 0)
3631       s->err.http_ret = redirect_code; // Apply a custom HTTP response code
3632     ldout(s->cct, 10) << "error handler redirect code=" << redirect_code
3633                       << " proto+host:" << protocol << "://" << hostname
3634                       << " -> " << s->redirect << dendl;
3635     return -ERR_WEBSITE_REDIRECT;
3636   } else if (err_no == -ERR_WEBSITE_REDIRECT) {
3637     // Do nothing here, this redirect will be handled in abort_early's ERR_WEBSITE_REDIRECT block
3638     // Do NOT fire the ErrorDoc handler
3639   } else if (!s->bucket_info.website_conf.error_doc.empty()) {
3640     /* This serves an entire page!
3641        On success, it will return zero, and no further content should be sent to the socket
3642        On failure, we need the double-error handler
3643      */
3644     new_err_no = RGWHandler_REST_S3Website::serve_errordoc(http_error_code, s->bucket_info.website_conf.error_doc);
3645     if (new_err_no && new_err_no != -1) {
3646       err_no = new_err_no;
3647     }
3648   } else {
3649     ldout(s->cct, 20) << "No special error handling today!" << dendl;
3650   }
3651
3652   return err_no;
3653 }
3654
3655 RGWOp* RGWHandler_REST_Obj_S3Website::get_obj_op(bool get_data)
3656 {
3657   /** If we are in website mode, then it is explicitly impossible to run GET or
3658    * HEAD on the actual directory. We must convert the request to run on the
3659    * suffix object instead!
3660    */
3661   RGWGetObj_ObjStore_S3Website* op = new RGWGetObj_ObjStore_S3Website;
3662   op->set_get_data(get_data);
3663   return op;
3664 }
3665
3666 RGWOp* RGWHandler_REST_Bucket_S3Website::get_obj_op(bool get_data)
3667 {
3668   /** If we are in website mode, then it is explicitly impossible to run GET or
3669    * HEAD on the actual directory. We must convert the request to run on the
3670    * suffix object instead!
3671    */
3672   RGWGetObj_ObjStore_S3Website* op = new RGWGetObj_ObjStore_S3Website;
3673   op->set_get_data(get_data);
3674   return op;
3675 }
3676
3677 RGWOp* RGWHandler_REST_Service_S3Website::get_obj_op(bool get_data)
3678 {
3679   /** If we are in website mode, then it is explicitly impossible to run GET or
3680    * HEAD on the actual directory. We must convert the request to run on the
3681    * suffix object instead!
3682    */
3683   RGWGetObj_ObjStore_S3Website* op = new RGWGetObj_ObjStore_S3Website;
3684   op->set_get_data(get_data);
3685   return op;
3686 }
3687
3688
3689 namespace rgw {
3690 namespace auth {
3691 namespace s3 {
3692
3693 static rgw::auth::Completer::cmplptr_t
3694 null_completer_factory(const boost::optional<std::string>& secret_key)
3695 {
3696   return nullptr;
3697 }
3698
3699
3700 AWSEngine::VersionAbstractor::auth_data_t
3701 AWSGeneralAbstractor::get_auth_data(const req_state* const s) const
3702 {
3703   AwsVersion version;
3704   AwsRoute route;
3705   std::tie(version, route) = discover_aws_flavour(s->info);
3706
3707   if (version == AwsVersion::V2) {
3708     return get_auth_data_v2(s);
3709   } else if (version == AwsVersion::V4) {
3710     return get_auth_data_v4(s, route == AwsRoute::QUERY_STRING);
3711   } else {
3712     /* FIXME(rzarzynski): handle anon user. */
3713     throw -EINVAL;
3714   }
3715 }
3716
3717 boost::optional<std::string>
3718 AWSGeneralAbstractor::get_v4_canonical_headers(
3719   const req_info& info,
3720   const boost::string_view& signedheaders,
3721   const bool using_qs) const
3722 {
3723   return rgw::auth::s3::get_v4_canonical_headers(info, signedheaders,
3724                                                  using_qs, false);
3725 }
3726
3727 AWSEngine::VersionAbstractor::auth_data_t
3728 AWSGeneralAbstractor::get_auth_data_v4(const req_state* const s,
3729                                        /* FIXME: const. */
3730                                        bool using_qs) const
3731 {
3732   boost::string_view access_key_id;
3733   boost::string_view signed_hdrs;
3734
3735   boost::string_view date;
3736   boost::string_view credential_scope;
3737   boost::string_view client_signature;
3738
3739   int ret = rgw::auth::s3::parse_credentials(s->info,
3740                                              access_key_id,
3741                                              credential_scope,
3742                                              signed_hdrs,
3743                                              client_signature,
3744                                              date,
3745                                              using_qs);
3746   if (ret < 0) {
3747     throw ret;
3748   }
3749
3750   /* craft canonical headers */
3751   boost::optional<std::string> canonical_headers = \
3752     get_v4_canonical_headers(s->info, signed_hdrs, using_qs);
3753   if (canonical_headers) {
3754     ldout(s->cct, 10) << "canonical headers format = " << *canonical_headers
3755                       << dendl;
3756   } else {
3757     throw -EPERM;
3758   }
3759
3760   /* Get the expected hash. */
3761   auto exp_payload_hash = rgw::auth::s3::get_v4_exp_payload_hash(s->info);
3762
3763   /* Craft canonical URI. Using std::move later so let it be non-const. */
3764   auto canonical_uri = rgw::auth::s3::get_v4_canonical_uri(s->info);
3765
3766   /* Craft canonical query string. std::moving later so non-const here. */
3767   auto canonical_qs = rgw::auth::s3::get_v4_canonical_qs(s->info, using_qs);
3768
3769   /* Craft canonical request. */
3770   auto canonical_req_hash = \
3771     rgw::auth::s3::get_v4_canon_req_hash(s->cct,
3772                                          s->info.method,
3773                                          std::move(canonical_uri),
3774                                          std::move(canonical_qs),
3775                                          std::move(*canonical_headers),
3776                                          signed_hdrs,
3777                                          exp_payload_hash);
3778
3779   auto string_to_sign = \
3780     rgw::auth::s3::get_v4_string_to_sign(s->cct,
3781                                          AWS4_HMAC_SHA256_STR,
3782                                          date,
3783                                          credential_scope,
3784                                          std::move(canonical_req_hash));
3785
3786   const auto sig_factory = std::bind(rgw::auth::s3::get_v4_signature,
3787                                      credential_scope,
3788                                      std::placeholders::_1,
3789                                      std::placeholders::_2,
3790                                      std::placeholders::_3);
3791
3792   /* Requests authenticated with the Query Parameters are treated as unsigned.
3793    * From "Authenticating Requests: Using Query Parameters (AWS Signature
3794    * Version 4)":
3795    *
3796    *   You don't include a payload hash in the Canonical Request, because
3797    *   when you create a presigned URL, you don't know the payload content
3798    *   because the URL is used to upload an arbitrary payload. Instead, you
3799    *   use a constant string UNSIGNED-PAYLOAD.
3800    *
3801    * This means we have absolutely no business in spawning completer. Both
3802    * aws4_auth_needs_complete and aws4_auth_streaming_mode are set to false
3803    * by default. We don't need to change that. */
3804   if (is_v4_payload_unsigned(exp_payload_hash) || is_v4_payload_empty(s)) {
3805     return {
3806       access_key_id,
3807       client_signature,
3808       std::move(string_to_sign),
3809       sig_factory,
3810       null_completer_factory
3811     };
3812   } else {
3813     /* We're going to handle a signed payload. Be aware that even empty HTTP
3814      * body (no payload) requires verification:
3815      *
3816      *   The x-amz-content-sha256 header is required for all AWS Signature
3817      *   Version 4 requests. It provides a hash of the request payload. If
3818      *   there is no payload, you must provide the hash of an empty string. */
3819     if (!is_v4_payload_streamed(exp_payload_hash)) {
3820       ldout(s->cct, 10) << "delaying v4 auth" << dendl;
3821
3822       /* payload in a single chunk */
3823       switch (s->op_type)
3824       {
3825         case RGW_OP_CREATE_BUCKET:
3826         case RGW_OP_PUT_OBJ:
3827         case RGW_OP_PUT_ACLS:
3828         case RGW_OP_PUT_CORS:
3829         case RGW_OP_INIT_MULTIPART: // in case that Init Multipart uses CHUNK encoding
3830         case RGW_OP_COMPLETE_MULTIPART:
3831         case RGW_OP_SET_BUCKET_VERSIONING:
3832         case RGW_OP_DELETE_MULTI_OBJ:
3833         case RGW_OP_ADMIN_SET_METADATA:
3834         case RGW_OP_SET_BUCKET_WEBSITE:
3835         case RGW_OP_PUT_BUCKET_POLICY:
3836         case RGW_OP_PUT_OBJ_TAGGING:
3837         case RGW_OP_PUT_LC:
3838         case RGW_OP_SET_REQUEST_PAYMENT:
3839           break;
3840         default:
3841           dout(10) << "ERROR: AWS4 completion for this operation NOT IMPLEMENTED" << dendl;
3842           throw -ERR_NOT_IMPLEMENTED;
3843       }
3844
3845       const auto cmpl_factory = std::bind(AWSv4ComplSingle::create,
3846                                           s,
3847                                           std::placeholders::_1);
3848       return {
3849         access_key_id,
3850         client_signature,
3851         std::move(string_to_sign),
3852         sig_factory,
3853         cmpl_factory
3854       };
3855     } else {
3856       /* IMHO "streamed" doesn't fit too good here. I would prefer to call
3857        * it "chunked" but let's be coherent with Amazon's terminology. */
3858
3859       dout(10) << "body content detected in multiple chunks" << dendl;
3860
3861       /* payload in multiple chunks */
3862
3863       switch(s->op_type)
3864       {
3865         case RGW_OP_PUT_OBJ:
3866           break;
3867         default:
3868           dout(10) << "ERROR: AWS4 completion for this operation NOT IMPLEMENTED (streaming mode)" << dendl;
3869           throw -ERR_NOT_IMPLEMENTED;
3870       }
3871
3872       dout(10) << "aws4 seed signature ok... delaying v4 auth" << dendl;
3873
3874       /* In the case of streamed payload client sets the x-amz-content-sha256
3875        * to "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" but uses "UNSIGNED-PAYLOAD"
3876        * when constructing the Canonical Request. */
3877
3878       /* In the case of single-chunk upload client set the header's value is
3879        * coherent with the one used for Canonical Request crafting. */
3880
3881       /* In the case of query string-based authentication there should be no
3882        * x-amz-content-sha256 header and the value "UNSIGNED-PAYLOAD" is used
3883        * for CanonReq. */
3884       const auto cmpl_factory = std::bind(AWSv4ComplMulti::create,
3885                                           s,
3886                                           date,
3887                                           credential_scope,
3888                                           client_signature,
3889                                           std::placeholders::_1);
3890       return {
3891         access_key_id,
3892         client_signature,
3893         std::move(string_to_sign),
3894         sig_factory,
3895         cmpl_factory
3896       };
3897     }
3898   }
3899 }
3900
3901
3902 boost::optional<std::string>
3903 AWSGeneralBoto2Abstractor::get_v4_canonical_headers(
3904   const req_info& info,
3905   const boost::string_view& signedheaders,
3906   const bool using_qs) const
3907 {
3908   return rgw::auth::s3::get_v4_canonical_headers(info, signedheaders,
3909                                                  using_qs, true);
3910 }
3911
3912
3913 AWSEngine::VersionAbstractor::auth_data_t
3914 AWSGeneralAbstractor::get_auth_data_v2(const req_state* const s) const
3915 {
3916   boost::string_view access_key_id;
3917   boost::string_view signature;
3918   bool qsr = false;
3919
3920   const char* http_auth = s->info.env->get("HTTP_AUTHORIZATION");
3921   if (! http_auth || http_auth[0] == '\0') {
3922     /* Credentials are provided in query string. We also need to verify
3923      * the "Expires" parameter now. */
3924     access_key_id = s->info.args.get("AWSAccessKeyId");
3925     signature = s->info.args.get("Signature");
3926     qsr = true;
3927
3928     boost::string_view expires = s->info.args.get("Expires");
3929     if (expires.empty()) {
3930       throw -EPERM;
3931     }
3932
3933     /* It looks we have the guarantee that expires is a null-terminated,
3934      * and thus string_view::data() can be safely used. */
3935     const time_t exp = atoll(expires.data());
3936     time_t now;
3937     time(&now);
3938
3939     if (now >= exp) {
3940       throw -EPERM;
3941     }
3942   } else {
3943     /* The "Authorization" HTTP header is being used. */
3944     const boost::string_view auth_str(http_auth + strlen("AWS "));
3945     const size_t pos = auth_str.rfind(':');
3946     if (pos != boost::string_view::npos) {
3947       access_key_id = auth_str.substr(0, pos);
3948       signature = auth_str.substr(pos + 1);
3949     }
3950   }
3951
3952   /* Let's canonize the HTTP headers that are covered by the AWS auth v2. */
3953   std::string string_to_sign;
3954   utime_t header_time;
3955   if (! rgw_create_s3_canonical_header(s->info, &header_time, string_to_sign,
3956         qsr)) {
3957     ldout(cct, 10) << "failed to create the canonized auth header\n"
3958                    << rgw::crypt_sanitize::auth{s,string_to_sign} << dendl;
3959     throw -EPERM;
3960   }
3961
3962   ldout(cct, 10) << "string_to_sign:\n"
3963                  << rgw::crypt_sanitize::auth{s,string_to_sign} << dendl;
3964
3965   if (!qsr && !is_time_skew_ok(header_time)) {
3966     throw -ERR_REQUEST_TIME_SKEWED;
3967   }
3968
3969   return {
3970     std::move(access_key_id),
3971     std::move(signature),
3972     std::move(string_to_sign),
3973     rgw::auth::s3::get_v2_signature,
3974     null_completer_factory
3975   };
3976 }
3977
3978
3979 AWSEngine::VersionAbstractor::auth_data_t
3980 AWSBrowserUploadAbstractor::get_auth_data_v2(const req_state* const s) const
3981 {
3982   return {
3983     s->auth.s3_postobj_creds.access_key,
3984     s->auth.s3_postobj_creds.signature,
3985     s->auth.s3_postobj_creds.encoded_policy.to_str(),
3986     rgw::auth::s3::get_v2_signature,
3987     null_completer_factory
3988   };
3989 }
3990
3991 AWSEngine::VersionAbstractor::auth_data_t
3992 AWSBrowserUploadAbstractor::get_auth_data_v4(const req_state* const s) const
3993 {
3994   const boost::string_view credential = s->auth.s3_postobj_creds.x_amz_credential;
3995
3996   /* grab access key id */
3997   const size_t pos = credential.find("/");
3998   const boost::string_view access_key_id = credential.substr(0, pos);
3999   dout(10) << "access key id = " << access_key_id << dendl;
4000
4001   /* grab credential scope */
4002   const boost::string_view credential_scope = credential.substr(pos + 1);
4003   dout(10) << "credential scope = " << credential_scope << dendl;
4004
4005   const auto sig_factory = std::bind(rgw::auth::s3::get_v4_signature,
4006                                      credential_scope,
4007                                      std::placeholders::_1,
4008                                      std::placeholders::_2,
4009                                      std::placeholders::_3);
4010
4011   return {
4012     access_key_id,
4013     s->auth.s3_postobj_creds.signature,
4014     s->auth.s3_postobj_creds.encoded_policy.to_str(),
4015     sig_factory,
4016     null_completer_factory
4017   };
4018 }
4019
4020 AWSEngine::VersionAbstractor::auth_data_t
4021 AWSBrowserUploadAbstractor::get_auth_data(const req_state* const s) const
4022 {
4023   if (s->auth.s3_postobj_creds.x_amz_algorithm == AWS4_HMAC_SHA256_STR) {
4024     ldout(s->cct, 0) << "Signature verification algorithm AWS v4"
4025                      << " (AWS4-HMAC-SHA256)" << dendl;
4026     return get_auth_data_v4(s);
4027   } else {
4028     ldout(s->cct, 0) << "Signature verification algorithm AWS v2" << dendl;
4029     return get_auth_data_v2(s);
4030   }
4031 }
4032
4033
4034 AWSEngine::result_t
4035 AWSEngine::authenticate(const req_state* const s) const
4036 {
4037   /* Small reminder: an ver_abstractor is allowed to throw! */
4038   const auto auth_data = ver_abstractor.get_auth_data(s);
4039
4040   if (auth_data.access_key_id.empty() || auth_data.client_signature.empty()) {
4041     return result_t::deny(-EINVAL);
4042   } else {
4043     return authenticate(auth_data.access_key_id,
4044                         auth_data.client_signature,
4045                         auth_data.string_to_sign,
4046                         auth_data.signature_factory,
4047                         auth_data.completer_factory,
4048                         s);
4049   }
4050 }
4051
4052 } /* namespace s3 */
4053 } /* namespace auth */
4054 } /* namespace rgw */
4055
4056 rgw::LDAPHelper* rgw::auth::s3::LDAPEngine::ldh = nullptr;
4057 std::mutex rgw::auth::s3::LDAPEngine::mtx;
4058
4059 void rgw::auth::s3::LDAPEngine::init(CephContext* const cct)
4060 {
4061   if (! ldh) {
4062     std::lock_guard<std::mutex> lck(mtx);
4063     if (! ldh) {
4064       const string& ldap_uri = cct->_conf->rgw_ldap_uri;
4065       const string& ldap_binddn = cct->_conf->rgw_ldap_binddn;
4066       const string& ldap_searchdn = cct->_conf->rgw_ldap_searchdn;
4067       const string& ldap_searchfilter = cct->_conf->rgw_ldap_searchfilter;
4068       const string& ldap_dnattr = cct->_conf->rgw_ldap_dnattr;
4069       std::string ldap_bindpw = parse_rgw_ldap_bindpw(cct);
4070
4071       ldh = new rgw::LDAPHelper(ldap_uri, ldap_binddn, ldap_bindpw,
4072                                 ldap_searchdn, ldap_searchfilter, ldap_dnattr);
4073
4074       ldh->init();
4075       ldh->bind();
4076     }
4077   }
4078 }
4079
4080 rgw::auth::RemoteApplier::acl_strategy_t
4081 rgw::auth::s3::LDAPEngine::get_acl_strategy() const
4082 {
4083   //This is based on the assumption that the default acl strategy in
4084   // get_perms_from_aclspec, will take care. Extra acl spec is not required.
4085   return nullptr;
4086 }
4087
4088 rgw::auth::RemoteApplier::AuthInfo
4089 rgw::auth::s3::LDAPEngine::get_creds_info(const rgw::RGWToken& token) const noexcept
4090 {
4091   /* The short form of "using" can't be used here -- we're aliasing a class'
4092    * member. */
4093   using acct_privilege_t = \
4094     rgw::auth::RemoteApplier::AuthInfo::acct_privilege_t;
4095
4096   return rgw::auth::RemoteApplier::AuthInfo {
4097     rgw_user(token.id),
4098     token.id,
4099     RGW_PERM_FULL_CONTROL,
4100     acct_privilege_t::IS_PLAIN_ACCT,
4101     TYPE_LDAP
4102   };
4103 }
4104
4105 rgw::auth::Engine::result_t
4106 rgw::auth::s3::LDAPEngine::authenticate(
4107   const boost::string_view& access_key_id,
4108   const boost::string_view& signature,
4109   const string_to_sign_t& string_to_sign,
4110   const signature_factory_t&,
4111   const completer_factory_t& completer_factory,
4112   const req_state* const s) const
4113 {
4114   /* boost filters and/or string_ref may throw on invalid input */
4115   rgw::RGWToken base64_token;
4116   try {
4117     base64_token = rgw::from_base64(access_key_id);
4118   } catch (...) {
4119     base64_token = std::string("");
4120   }
4121
4122   if (! base64_token.valid()) {
4123     return result_t::deny();
4124   }
4125
4126   //TODO: Uncomment, when we have a migration plan in place.
4127   //Check if a user of type other than 'ldap' is already present, if yes, then
4128   //return error.
4129   /*RGWUserInfo user_info;
4130   user_info.user_id = base64_token.id;
4131   if (rgw_get_user_info_by_uid(store, user_info.user_id, user_info) >= 0) {
4132     if (user_info.type != TYPE_LDAP) {
4133       ldout(cct, 10) << "ERROR: User id of type: " << user_info.type << " is already present" << dendl;
4134       return nullptr;
4135     }
4136   }*/
4137
4138   if (ldh->auth(base64_token.id, base64_token.key) != 0) {
4139     return result_t::deny();
4140   }
4141
4142   auto apl = apl_factory->create_apl_remote(cct, s, get_acl_strategy(),
4143                                             get_creds_info(base64_token));
4144   return result_t::grant(std::move(apl), completer_factory(boost::none));
4145 }
4146
4147
4148 /* LocalEndgine */
4149 rgw::auth::Engine::result_t
4150 rgw::auth::s3::LocalEngine::authenticate(
4151   const boost::string_view& _access_key_id,
4152   const boost::string_view& signature,
4153   const string_to_sign_t& string_to_sign,
4154   const signature_factory_t& signature_factory,
4155   const completer_factory_t& completer_factory,
4156   const req_state* const s) const
4157 {
4158   /* get the user info */
4159   RGWUserInfo user_info;
4160   /* TODO(rzarzynski): we need to have string-view taking variant. */
4161   const std::string access_key_id = _access_key_id.to_string();
4162   if (rgw_get_user_info_by_access_key(store, access_key_id, user_info) < 0) {
4163       ldout(cct, 5) << "error reading user info, uid=" << access_key_id
4164               << " can't authenticate" << dendl;
4165       return result_t::deny(-ERR_INVALID_ACCESS_KEY);
4166   }
4167   //TODO: Uncomment, when we have a migration plan in place.
4168   /*else {
4169     if (s->user->type != TYPE_RGW) {
4170       ldout(cct, 10) << "ERROR: User id of type: " << s->user->type
4171                      << " is present" << dendl;
4172       throw -EPERM;
4173     }
4174   }*/
4175
4176   const auto iter = user_info.access_keys.find(access_key_id);
4177   if (iter == std::end(user_info.access_keys)) {
4178     ldout(cct, 0) << "ERROR: access key not encoded in user info" << dendl;
4179     return result_t::deny(-EPERM);
4180   }
4181   const RGWAccessKey& k = iter->second;
4182
4183   const VersionAbstractor::server_signature_t server_signature = \
4184     signature_factory(cct, k.key, string_to_sign);
4185
4186   ldout(cct, 15) << "string_to_sign="
4187                  << rgw::crypt_sanitize::log_content{string_to_sign}
4188                  << dendl;
4189   ldout(cct, 15) << "server signature=" << server_signature << dendl;
4190   ldout(cct, 15) << "client signature=" << signature << dendl;
4191   ldout(cct, 15) << "compare=" << signature.compare(server_signature) << dendl;
4192
4193   if (static_cast<boost::string_view>(server_signature) != signature) {
4194     return result_t::deny(-ERR_SIGNATURE_NO_MATCH);
4195   }
4196
4197   auto apl = apl_factory->create_apl_local(cct, s, user_info, k.subuser);
4198   return result_t::grant(std::move(apl), completer_factory(k.key));
4199 }
4200
4201 bool rgw::auth::s3::S3AnonymousEngine::is_applicable(
4202   const req_state* s
4203 ) const noexcept {
4204   if (s->op == OP_OPTIONS) {
4205     return true;
4206   }
4207
4208   AwsVersion version;
4209   AwsRoute route;
4210   std::tie(version, route) = discover_aws_flavour(s->info);
4211
4212   return route == AwsRoute::QUERY_STRING && version == AwsVersion::UNKOWN;
4213 }