ceph/src/rgw/rgw_rest_s3.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3
   4 #include <errno.h>
   5 #include <array>
   6 #include <string.h>
   7
   8 #include "common/ceph_crypto.h"
   9 #include "common/Formatter.h"
  10 #include "common/utf8.h"
  11 #include "common/ceph_json.h"
  12 #include "common/safe_io.h"
  13 #include "common/backport14.h"
  14 #include <boost/algorithm/string.hpp>
  15 #include <boost/algorithm/string/replace.hpp>
  16 #include <boost/utility/string_view.hpp>
  17
  18 #include "rgw_rest.h"
  19 #include "rgw_rest_s3.h"
  20 #include "rgw_rest_s3website.h"
  21 #include "rgw_auth_s3.h"
  22 #include "rgw_acl.h"
  23 #include "rgw_policy_s3.h"
  24 #include "rgw_user.h"
  25 #include "rgw_cors.h"
  26 #include "rgw_cors_s3.h"
  27 #include "rgw_tag_s3.h"
  28
  29 #include "rgw_client_io.h"
  30
  31 #include "rgw_keystone.h"
  32 #include "rgw_auth_keystone.h"
  33 #include "rgw_auth_registry.h"
  34
  35 #include "rgw_es_query.h"
  36
  37 #include <typeinfo> // for 'typeid'
  38
  39 #include "rgw_ldap.h"
  40 #include "rgw_token.h"
  41 #include "rgw_rest_role.h"
  42 #include "rgw_crypt.h"
  43 #include "rgw_crypt_sanitize.h"
  44
  45 #include "include/assert.h"
  46
  47 #define dout_context g_ceph_context
  48 #define dout_subsys ceph_subsys_rgw
  49
  50 using namespace rgw;
  51 using namespace ceph::crypto;
  52
  53 using std::get;
  54
  55 void list_all_buckets_start(struct req_state *s)
  56 {
  57   s->formatter->open_array_section_in_ns("ListAllMyBucketsResult", XMLNS_AWS_S3);
  58 }
  59
  60 void list_all_buckets_end(struct req_state *s)
  61 {
  62   s->formatter->close_section();
  63 }
  64
  65 void dump_bucket(struct req_state *s, RGWBucketEnt& obj)
  66 {
  67   s->formatter->open_object_section("Bucket");
  68   s->formatter->dump_string("Name", obj.bucket.name);
  69   dump_time(s, "CreationDate", &obj.creation_time);
  70   s->formatter->close_section();
  71 }
  72
  73 void rgw_get_errno_s3(rgw_http_error *e , int err_no)
  74 {
  75   rgw_http_errors::const_iterator r = rgw_http_s3_errors.find(err_no);
  76
  77   if (r != rgw_http_s3_errors.end()) {
  78     e->http_ret = r->second.first;
  79     e->s3_code = r->second.second;
  80   } else {
  81     e->http_ret = 500;
  82     e->s3_code = "UnknownError";
  83   }
  84 }
  85
  86 struct response_attr_param {
  87   const char *param;
  88   const char *http_attr;
  89 };
  90
  91 static struct response_attr_param resp_attr_params[] = {
  92   {"response-content-type", "Content-Type"},
  93   {"response-content-language", "Content-Language"},
  94   {"response-expires", "Expires"},
  95   {"response-cache-control", "Cache-Control"},
  96   {"response-content-disposition", "Content-Disposition"},
  97   {"response-content-encoding", "Content-Encoding"},
  98   {NULL, NULL},
  99 };
 100
 101 int RGWGetObj_ObjStore_S3Website::send_response_data(bufferlist& bl, off_t bl_ofs, off_t bl_len) {
 102   map<string, bufferlist>::iterator iter;
 103   iter = attrs.find(RGW_ATTR_AMZ_WEBSITE_REDIRECT_LOCATION);
 104   if (iter != attrs.end()) {
 105     bufferlist &bl = iter->second;
 106     s->redirect = string(bl.c_str(), bl.length());
 107     s->err.http_ret = 301;
 108     ldout(s->cct, 20) << __CEPH_ASSERT_FUNCTION << " redirecting per x-amz-website-redirect-location=" << s->redirect << dendl;
 109     op_ret = -ERR_WEBSITE_REDIRECT;
 110     set_req_state_err(s, op_ret);
 111     dump_errno(s);
 112     dump_content_length(s, 0);
 113     dump_redirect(s, s->redirect);
 114     end_header(s, this);
 115     return op_ret;
 116   } else {
 117     return RGWGetObj_ObjStore_S3::send_response_data(bl, bl_ofs, bl_len);
 118   }
 119 }
 120
 121 int RGWGetObj_ObjStore_S3Website::send_response_data_error()
 122 {
 123   return RGWGetObj_ObjStore_S3::send_response_data_error();
 124 }
 125
 126 int RGWGetObj_ObjStore_S3::get_params()
 127 {
 128   // for multisite sync requests, only read the slo manifest itself, rather than
 129   // all of the data from its parts. the parts will sync as separate objects
 130   skip_manifest = s->info.args.exists(RGW_SYS_PARAM_PREFIX "sync-manifest");
 131
 132   // multisite sync requests should fetch encrypted data, along with the
 133   // attributes needed to support decryption on the other zone
 134   if (s->system_request) {
 135     skip_decrypt = s->info.args.exists(RGW_SYS_PARAM_PREFIX "skip-decrypt");
 136   }
 137
 138   return RGWGetObj_ObjStore::get_params();
 139 }
 140
 141 int RGWGetObj_ObjStore_S3::send_response_data_error()
 142 {
 143   bufferlist bl;
 144   return send_response_data(bl, 0 , 0);
 145 }
 146
 147 template <class T>
 148 int decode_attr_bl_single_value(map<string, bufferlist>& attrs, const char *attr_name, T *result, T def_val)
 149 {
 150   map<string, bufferlist>::iterator iter = attrs.find(attr_name);
 151   if (iter == attrs.end()) {
 152     *result = def_val;
 153     return 0;
 154   }
 155   bufferlist& bl = iter->second;
 156   if (bl.length() == 0) {
 157     *result = def_val;
 158     return 0;
 159   }
 160   bufferlist::iterator bliter = bl.begin();
 161   try {
 162     ::decode(*result, bliter);
 163   } catch (buffer::error& err) {
 164     return -EIO;
 165   }
 166   return 0;
 167 }
 168
 169 int RGWGetObj_ObjStore_S3::send_response_data(bufferlist& bl, off_t bl_ofs,
 170                                               off_t bl_len)
 171 {
 172   const char *content_type = NULL;
 173   string content_type_str;
 174   map<string, string> response_attrs;
 175   map<string, string>::iterator riter;
 176   bufferlist metadata_bl;
 177
 178   if (sent_header)
 179     goto send_data;
 180
 181   if (custom_http_ret) {
 182     set_req_state_err(s, 0);
 183     dump_errno(s, custom_http_ret);
 184   } else {
 185     set_req_state_err(s, (partial_content && !op_ret) ? STATUS_PARTIAL_CONTENT
 186                   : op_ret);
 187     dump_errno(s);
 188   }
 189
 190   if (op_ret)
 191     goto done;
 192
 193   if (range_str)
 194     dump_range(s, start, end, s->obj_size);
 195
 196   if (s->system_request &&
 197       s->info.args.exists(RGW_SYS_PARAM_PREFIX "prepend-metadata")) {
 198
 199     dump_header(s, "Rgwx-Object-Size", (long long)total_len);
 200
 201     if (rgwx_stat) {
 202       /*
 203        * in this case, we're not returning the object's content, only the prepended
 204        * extra metadata
 205        */
 206       total_len = 0;
 207     }
 208
 209     /* JSON encode object metadata */
 210     JSONFormatter jf;
 211     jf.open_object_section("obj_metadata");
 212     encode_json("attrs", attrs, &jf);
 213     utime_t ut(lastmod);
 214     encode_json("mtime", ut, &jf);
 215     jf.close_section();
 216     stringstream ss;
 217     jf.flush(ss);
 218     metadata_bl.append(ss.str());
 219     dump_header(s, "Rgwx-Embedded-Metadata-Len", metadata_bl.length());
 220     total_len += metadata_bl.length();
 221   }
 222
 223   if (s->system_request && !real_clock::is_zero(lastmod)) {
 224     /* we end up dumping mtime in two different methods, a bit redundant */
 225     dump_epoch_header(s, "Rgwx-Mtime", lastmod);
 226     uint64_t pg_ver = 0;
 227     int r = decode_attr_bl_single_value(attrs, RGW_ATTR_PG_VER, &pg_ver, (uint64_t)0);
 228     if (r < 0) {
 229       ldout(s->cct, 0) << "ERROR: failed to decode pg ver attr, ignoring" << dendl;
 230     }
 231     dump_header(s, "Rgwx-Obj-PG-Ver", pg_ver);
 232
 233     uint32_t source_zone_short_id = 0;
 234     r = decode_attr_bl_single_value(attrs, RGW_ATTR_SOURCE_ZONE, &source_zone_short_id, (uint32_t)0);
 235     if (r < 0) {
 236       ldout(s->cct, 0) << "ERROR: failed to decode pg ver attr, ignoring" << dendl;
 237     }
 238     if (source_zone_short_id != 0) {
 239       dump_header(s, "Rgwx-Source-Zone-Short-Id", source_zone_short_id);
 240     }
 241   }
 242
 243   for (auto &it : crypt_http_responses)
 244     dump_header(s, it.first, it.second);
 245
 246   dump_content_length(s, total_len);
 247   dump_last_modified(s, lastmod);
 248   if (!version_id.empty()) {
 249     dump_header(s, "x-amz-version-id", version_id);
 250   }
 251
 252
 253   if (! op_ret) {
 254     if (! lo_etag.empty()) {
 255       /* Handle etag of Swift API's large objects (DLO/SLO). It's entirerly
 256        * legit to perform GET on them through S3 API. In such situation,
 257        * a client should receive the composited content with corresponding
 258        * etag value. */
 259       dump_etag(s, lo_etag);
 260     } else {
 261       auto iter = attrs.find(RGW_ATTR_ETAG);
 262       if (iter != attrs.end()) {
 263         dump_etag(s, iter->second);
 264       }
 265     }
 266
 267     for (struct response_attr_param *p = resp_attr_params; p->param; p++) {
 268       bool exists;
 269       string val = s->info.args.get(p->param, &exists);
 270       if (exists) {
 271         if (strcmp(p->param, "response-content-type") != 0) {
 272           response_attrs[p->http_attr] = val;
 273         } else {
 274           content_type_str = val;
 275           content_type = content_type_str.c_str();
 276         }
 277       }
 278     }
 279
 280     for (auto iter = attrs.begin(); iter != attrs.end(); ++iter) {
 281       const char *name = iter->first.c_str();
 282       map<string, string>::iterator aiter = rgw_to_http_attrs.find(name);
 283       if (aiter != rgw_to_http_attrs.end()) {
 284         if (response_attrs.count(aiter->second) == 0) {
 285           /* Was not already overridden by a response param. */
 286           response_attrs[aiter->second] = iter->second.c_str();
 287         }
 288       } else if (iter->first.compare(RGW_ATTR_CONTENT_TYPE) == 0) {
 289         /* Special handling for content_type. */
 290         if (!content_type) {
 291           content_type = iter->second.c_str();
 292         }
 293       } else if (strcmp(name, RGW_ATTR_SLO_UINDICATOR) == 0) {
 294         // this attr has an extra length prefix from ::encode() in prior versions
 295         dump_header(s, "X-Object-Meta-Static-Large-Object", "True");
 296       } else if (strncmp(name, RGW_ATTR_META_PREFIX,
 297                          sizeof(RGW_ATTR_META_PREFIX)-1) == 0) {
 298         /* User custom metadata. */
 299         name += sizeof(RGW_ATTR_PREFIX) - 1;
 300         dump_header(s, name, iter->second);
 301       } else if (iter->first.compare(RGW_ATTR_TAGS) == 0) {
 302         RGWObjTags obj_tags;
 303         try{
 304           bufferlist::iterator it = iter->second.begin();
 305           obj_tags.decode(it);
 306         } catch (buffer::error &err) {
 307           ldout(s->cct,0) << "Error caught buffer::error couldn't decode TagSet " << dendl;
 308         }
 309         dump_header(s, RGW_AMZ_TAG_COUNT, obj_tags.count());
 310       }
 311     }
 312   }
 313
 314 done:
 315   for (riter = response_attrs.begin(); riter != response_attrs.end();
 316        ++riter) {
 317     dump_header(s, riter->first, riter->second);
 318   }
 319
 320   if (op_ret == -ERR_NOT_MODIFIED) {
 321       end_header(s, this);
 322   } else {
 323       if (!content_type)
 324           content_type = "binary/octet-stream";
 325
 326       end_header(s, this, content_type);
 327   }
 328
 329   if (metadata_bl.length()) {
 330     dump_body(s, metadata_bl);
 331   }
 332   sent_header = true;
 333
 334 send_data:
 335   if (get_data && !op_ret) {
 336     int r = dump_body(s, bl.c_str() + bl_ofs, bl_len);
 337     if (r < 0)
 338       return r;
 339   }
 340
 341   return 0;
 342 }
 343
 344 int RGWGetObj_ObjStore_S3::get_decrypt_filter(std::unique_ptr<RGWGetDataCB> *filter, RGWGetDataCB* cb, bufferlist* manifest_bl)
 345 {
 346   if (skip_decrypt) { // bypass decryption for multisite sync requests
 347     return 0;
 348   }
 349
 350   int res = 0;
 351   std::unique_ptr<BlockCrypt> block_crypt;
 352   res = rgw_s3_prepare_decrypt(s, attrs, &block_crypt, crypt_http_responses);
 353   if (res == 0) {
 354     if (block_crypt != nullptr) {
 355       auto f = std::unique_ptr<RGWGetObj_BlockDecrypt>(new RGWGetObj_BlockDecrypt(s->cct, cb, std::move(block_crypt)));
 356       //RGWGetObj_BlockDecrypt* f = new RGWGetObj_BlockDecrypt(s->cct, cb, std::move(block_crypt));
 357       if (f != nullptr) {
 358         if (manifest_bl != nullptr) {
 359           res = f->read_manifest(*manifest_bl);
 360           if (res == 0) {
 361             *filter = std::move(f);
 362           }
 363         }
 364       }
 365     }
 366   }
 367   return res;
 368 }
 369
 370 void RGWGetObjTags_ObjStore_S3::send_response_data(bufferlist& bl)
 371 {
 372   dump_errno(s);
 373   end_header(s, this, "application/xml");
 374   dump_start(s);
 375
 376   s->formatter->open_object_section_in_ns("Tagging", XMLNS_AWS_S3);
 377   s->formatter->open_object_section("TagSet");
 378   if (has_tags){
 379     RGWObjTagSet_S3 tagset;
 380     bufferlist::iterator iter = bl.begin();
 381     try {
 382       tagset.decode(iter);
 383     } catch (buffer::error& err) {
 384       ldout(s->cct,0) << "ERROR: caught buffer::error, couldn't decode TagSet" << dendl;
 385       op_ret= -EIO;
 386       return;
 387     }
 388     tagset.dump_xml(s->formatter);
 389   }
 390   s->formatter->close_section();
 391   s->formatter->close_section();
 392   rgw_flush_formatter_and_reset(s, s->formatter);
 393 }
 394
 395
 396 int RGWPutObjTags_ObjStore_S3::get_params()
 397 {
 398   RGWObjTagsXMLParser parser;
 399
 400   if (!parser.init()){
 401     return -EINVAL;
 402   }
 403
 404   char *data=nullptr;
 405   int len=0;
 406
 407   const auto max_size = s->cct->_conf->rgw_max_put_param_size;
 408   int r = rgw_rest_read_all_input(s, &data, &len, max_size, false);
 409
 410   if (r < 0)
 411     return r;
 412
 413   auto data_deleter = std::unique_ptr<char, decltype(free)*>{data, free};
 414
 415   if (!parser.parse(data, len, 1)) {
 416     return -ERR_MALFORMED_XML;
 417   }
 418
 419   RGWObjTagSet_S3 *obj_tags_s3;
 420   RGWObjTagging_S3 *tagging;
 421
 422   tagging = static_cast<RGWObjTagging_S3 *>(parser.find_first("Tagging"));
 423   obj_tags_s3 = static_cast<RGWObjTagSet_S3 *>(tagging->find_first("TagSet"));
 424   if(!obj_tags_s3){
 425     return -ERR_MALFORMED_XML;
 426   }
 427
 428   RGWObjTags obj_tags;
 429   r = obj_tags_s3->rebuild(obj_tags);
 430   if (r < 0)
 431     return r;
 432
 433   obj_tags.encode(tags_bl);
 434   ldout(s->cct, 20) << "Read " << obj_tags.count() << "tags" << dendl;
 435
 436   return 0;
 437 }
 438
 439 void RGWPutObjTags_ObjStore_S3::send_response()
 440 {
 441   if (op_ret)
 442     set_req_state_err(s, op_ret);
 443   dump_errno(s);
 444   end_header(s, this, "application/xml");
 445   dump_start(s);
 446
 447 }
 448
 449 void RGWDeleteObjTags_ObjStore_S3::send_response()
 450 {
 451   int r = op_ret;
 452   if (r == -ENOENT)
 453     r = 0;
 454   if (!r)
 455     r = STATUS_NO_CONTENT;
 456
 457   set_req_state_err(s, r);
 458   dump_errno(s);
 459   end_header(s, this);
 460 }
 461
 462 void RGWListBuckets_ObjStore_S3::send_response_begin(bool has_buckets)
 463 {
 464   if (op_ret)
 465     set_req_state_err(s, op_ret);
 466   dump_errno(s);
 467   dump_start(s);
 468   end_header(s, NULL, "application/xml");
 469
 470   if (! op_ret) {
 471     list_all_buckets_start(s);
 472     dump_owner(s, s->user->user_id, s->user->display_name);
 473     s->formatter->open_array_section("Buckets");
 474     sent_data = true;
 475   }
 476 }
 477
 478 void RGWListBuckets_ObjStore_S3::send_response_data(RGWUserBuckets& buckets)
 479 {
 480   if (!sent_data)
 481     return;
 482
 483   map<string, RGWBucketEnt>& m = buckets.get_buckets();
 484   map<string, RGWBucketEnt>::iterator iter;
 485
 486   for (iter = m.begin(); iter != m.end(); ++iter) {
 487     RGWBucketEnt obj = iter->second;
 488     dump_bucket(s, obj);
 489   }
 490   rgw_flush_formatter(s, s->formatter);
 491 }
 492
 493 void RGWListBuckets_ObjStore_S3::send_response_end()
 494 {
 495   if (sent_data) {
 496     s->formatter->close_section();
 497     list_all_buckets_end(s);
 498     rgw_flush_formatter_and_reset(s, s->formatter);
 499   }
 500 }
 501
 502 int RGWGetUsage_ObjStore_S3::get_params()
 503 {
 504   start_date = s->info.args.get("start-date");
 505   end_date = s->info.args.get("end-date");
 506   return 0;
 507 }
 508
 509 static void dump_usage_categories_info(Formatter *formatter, const rgw_usage_log_entry& entry, map<string, bool> *categories)
 510 {
 511   formatter->open_array_section("categories");
 512   map<string, rgw_usage_data>::const_iterator uiter;
 513   for (uiter = entry.usage_map.begin(); uiter != entry.usage_map.end(); ++uiter) {
 514     if (categories && !categories->empty() && !categories->count(uiter->first))
 515       continue;
 516     const rgw_usage_data& usage = uiter->second;
 517     formatter->open_object_section("Entry");
 518     formatter->dump_string("Category", uiter->first);
 519     formatter->dump_int("BytesSent", usage.bytes_sent);
 520     formatter->dump_int("BytesReceived", usage.bytes_received);
 521     formatter->dump_int("Ops", usage.ops);
 522     formatter->dump_int("SuccessfulOps", usage.successful_ops);
 523     formatter->close_section(); // Entry
 524   }
 525   formatter->close_section(); // Category
 526 }
 527
 528 static void dump_usage_bucket_info(Formatter *formatter, const std::string& name, const cls_user_bucket_entry& entry)
 529 {
 530   formatter->open_object_section("Entry");
 531   formatter->dump_string("Bucket", name);
 532   formatter->dump_int("Bytes", entry.size);
 533   formatter->dump_int("Bytes_Rounded", entry.size_rounded);
 534   formatter->close_section(); // entry
 535 }
 536
 537 void RGWGetUsage_ObjStore_S3::send_response()
 538 {
 539   if (op_ret < 0)
 540     set_req_state_err(s, op_ret);
 541   dump_errno(s);
 542
 543   end_header(s, this, "application/xml");
 544   dump_start(s);
 545   if (op_ret < 0)
 546     return;
 547
 548   Formatter *formatter = s->formatter;
 549   string last_owner;
 550   bool user_section_open = false;
 551
 552   formatter->open_object_section("Usage");
 553   if (show_log_entries) {
 554     formatter->open_array_section("Entries");
 555   }
 556   map<rgw_user_bucket, rgw_usage_log_entry>::iterator iter;
 557   for (iter = usage.begin(); iter != usage.end(); ++iter) {
 558     const rgw_user_bucket& ub = iter->first;
 559     const rgw_usage_log_entry& entry = iter->second;
 560
 561     if (show_log_entries) {
 562       if (ub.user.compare(last_owner) != 0) {
 563         if (user_section_open) {
 564           formatter->close_section();
 565           formatter->close_section();
 566         }
 567         formatter->open_object_section("User");
 568         formatter->dump_string("Owner", ub.user);
 569         formatter->open_array_section("Buckets");
 570         user_section_open = true;
 571         last_owner = ub.user;
 572       }
 573       formatter->open_object_section("Bucket");
 574       formatter->dump_string("Bucket", ub.bucket);
 575       utime_t ut(entry.epoch, 0);
 576       ut.gmtime(formatter->dump_stream("Time"));
 577       formatter->dump_int("Epoch", entry.epoch);
 578       dump_usage_categories_info(formatter, entry, &categories);
 579       formatter->close_section(); // bucket
 580     }
 581
 582     summary_map[ub.user].aggregate(entry, &categories);
 583   }
 584
 585   if (show_log_entries) {
 586      if (user_section_open) {
 587        formatter->close_section(); // buckets
 588        formatter->close_section(); //user
 589      }
 590      formatter->close_section(); // entries
 591    }
 592
 593    if (show_log_sum) {
 594      formatter->open_array_section("Summary");
 595      map<string, rgw_usage_log_entry>::iterator siter;
 596      for (siter = summary_map.begin(); siter != summary_map.end(); ++siter) {
 597        const rgw_usage_log_entry& entry = siter->second;
 598        formatter->open_object_section("User");
 599        formatter->dump_string("User", siter->first);
 600        dump_usage_categories_info(formatter, entry, &categories);
 601        rgw_usage_data total_usage;
 602        entry.sum(total_usage, categories);
 603        formatter->open_object_section("Total");
 604        formatter->dump_int("BytesSent", total_usage.bytes_sent);
 605        formatter->dump_int("BytesReceived", total_usage.bytes_received);
 606        formatter->dump_int("Ops", total_usage.ops);
 607        formatter->dump_int("SuccessfulOps", total_usage.successful_ops);
 608        formatter->close_section(); // total
 609        formatter->close_section(); // user
 610      }
 611
 612      if (s->cct->_conf->rgw_rest_getusage_op_compat) {
 613        formatter->open_object_section("Stats");
 614      }
 615
 616      formatter->dump_int("TotalBytes", header.stats.total_bytes);
 617      formatter->dump_int("TotalBytesRounded", header.stats.total_bytes_rounded);
 618      formatter->dump_int("TotalEntries", header.stats.total_entries);
 619
 620      if (s->cct->_conf->rgw_rest_getusage_op_compat) {
 621        formatter->close_section(); //Stats
 622      }
 623
 624      formatter->close_section(); // summary
 625    }
 626
 627   formatter->open_array_section("CapacityUsed");
 628   formatter->open_object_section("User");
 629   formatter->open_array_section("Buckets");
 630   for (const auto& biter : buckets_usage) {
 631     const cls_user_bucket_entry& entry = biter.second;
 632     dump_usage_bucket_info(formatter, biter.first, entry);
 633   }
 634   formatter->close_section(); // Buckets
 635   formatter->close_section(); // User
 636   formatter->close_section(); // CapacityUsed
 637
 638   formatter->close_section(); // usage
 639   rgw_flush_formatter_and_reset(s, s->formatter);
 640 }
 641
 642 int RGWListBucket_ObjStore_S3::get_params()
 643 {
 644   list_versions = s->info.args.exists("versions");
 645   prefix = s->info.args.get("prefix");
 646   if (!list_versions) {
 647     marker = s->info.args.get("marker");
 648   } else {
 649     marker.name = s->info.args.get("key-marker");
 650     marker.instance = s->info.args.get("version-id-marker");
 651   }
 652   max_keys = s->info.args.get("max-keys");
 653   op_ret = parse_max_keys();
 654   if (op_ret < 0) {
 655     return op_ret;
 656   }
 657   delimiter = s->info.args.get("delimiter");
 658   encoding_type = s->info.args.get("encoding-type");
 659   if (s->system_request) {
 660     s->info.args.get_bool("objs-container", &objs_container, false);
 661     const char *shard_id_str = s->info.env->get("HTTP_RGWX_SHARD_ID");
 662     if (shard_id_str) {
 663       string err;
 664       shard_id = strict_strtol(shard_id_str, 10, &err);
 665       if (!err.empty()) {
 666         ldout(s->cct, 5) << "bad shard id specified: " << shard_id_str << dendl;
 667         return -EINVAL;
 668       }
 669     } else {
 670       shard_id = s->bucket_instance_shard_id;
 671     }
 672   }
 673   return 0;
 674 }
 675
 676 void RGWListBucket_ObjStore_S3::send_versioned_response()
 677 {
 678   s->formatter->open_object_section_in_ns("ListVersionsResult", XMLNS_AWS_S3);
 679   if (!s->bucket_tenant.empty())
 680     s->formatter->dump_string("Tenant", s->bucket_tenant);
 681   s->formatter->dump_string("Name", s->bucket_name);
 682   s->formatter->dump_string("Prefix", prefix);
 683   s->formatter->dump_string("KeyMarker", marker.name);
 684   s->formatter->dump_string("VersionIdMarker", marker.instance);
 685   if (is_truncated && !next_marker.empty()) {
 686     s->formatter->dump_string("NextKeyMarker", next_marker.name);
 687     s->formatter->dump_string("NextVersionIdMarker", next_marker.instance);
 688   }
 689   s->formatter->dump_int("MaxKeys", max);
 690   if (!delimiter.empty())
 691     s->formatter->dump_string("Delimiter", delimiter);
 692
 693   s->formatter->dump_string("IsTruncated", (max && is_truncated ? "true"
 694                                             : "false"));
 695
 696   bool encode_key = false;
 697   if (strcasecmp(encoding_type.c_str(), "url") == 0) {
 698     s->formatter->dump_string("EncodingType", "url");
 699     encode_key = true;
 700   }
 701
 702   if (op_ret >= 0) {
 703     if (objs_container) {
 704       s->formatter->open_array_section("Entries");
 705     }
 706
 707     vector<rgw_bucket_dir_entry>::iterator iter;
 708     for (iter = objs.begin(); iter != objs.end(); ++iter) {
 709       const char *section_name = (iter->is_delete_marker() ? "DeleteMarker"
 710                                   : "Version");
 711       s->formatter->open_object_section(section_name);
 712       if (objs_container) {
 713         s->formatter->dump_bool("IsDeleteMarker", iter->is_delete_marker());
 714       }
 715       rgw_obj_key key(iter->key);
 716       if (encode_key) {
 717         string key_name;
 718         url_encode(key.name, key_name);
 719         s->formatter->dump_string("Key", key_name);
 720       } else {
 721         s->formatter->dump_string("Key", key.name);
 722       }
 723       string version_id = key.instance;
 724       if (version_id.empty()) {
 725         version_id = "null";
 726       }
 727       if (s->system_request) {
 728         if (iter->versioned_epoch > 0) {
 729           s->formatter->dump_int("VersionedEpoch", iter->versioned_epoch);
 730         }
 731         s->formatter->dump_string("RgwxTag", iter->tag);
 732         utime_t ut(iter->meta.mtime);
 733         ut.gmtime_nsec(s->formatter->dump_stream("RgwxMtime"));
 734       }
 735       s->formatter->dump_string("VersionId", version_id);
 736       s->formatter->dump_bool("IsLatest", iter->is_current());
 737       dump_time(s, "LastModified", &iter->meta.mtime);
 738       if (!iter->is_delete_marker()) {
 739         s->formatter->dump_format("ETag", "\"%s\"", iter->meta.etag.c_str());
 740         s->formatter->dump_int("Size", iter->meta.accounted_size);
 741         s->formatter->dump_string("StorageClass", "STANDARD");
 742       }
 743       dump_owner(s, iter->meta.owner, iter->meta.owner_display_name);
 744       s->formatter->close_section();
 745     }
 746     if (objs_container) {
 747       s->formatter->close_section();
 748     }
 749
 750     if (!common_prefixes.empty()) {
 751       map<string, bool>::iterator pref_iter;
 752       for (pref_iter = common_prefixes.begin();
 753            pref_iter != common_prefixes.end(); ++pref_iter) {
 754         s->formatter->open_array_section("CommonPrefixes");
 755         s->formatter->dump_string("Prefix", pref_iter->first);
 756         s->formatter->close_section();
 757       }
 758     }
 759   }
 760   s->formatter->close_section();
 761   rgw_flush_formatter_and_reset(s, s->formatter);
 762 }
 763
 764 void RGWListBucket_ObjStore_S3::send_response()
 765 {
 766   if (op_ret < 0)
 767     set_req_state_err(s, op_ret);
 768   dump_errno(s);
 769
 770   end_header(s, this, "application/xml");
 771   dump_start(s);
 772   if (op_ret < 0)
 773     return;
 774
 775   if (list_versions) {
 776     send_versioned_response();
 777     return;
 778   }
 779
 780   s->formatter->open_object_section_in_ns("ListBucketResult", XMLNS_AWS_S3);
 781   if (!s->bucket_tenant.empty())
 782     s->formatter->dump_string("Tenant", s->bucket_tenant);
 783   s->formatter->dump_string("Name", s->bucket_name);
 784   s->formatter->dump_string("Prefix", prefix);
 785   s->formatter->dump_string("Marker", marker.name);
 786   if (is_truncated && !next_marker.empty())
 787     s->formatter->dump_string("NextMarker", next_marker.name);
 788   s->formatter->dump_int("MaxKeys", max);
 789   if (!delimiter.empty())
 790     s->formatter->dump_string("Delimiter", delimiter);
 791
 792   s->formatter->dump_string("IsTruncated", (max && is_truncated ? "true"
 793                                             : "false"));
 794
 795   bool encode_key = false;
 796   if (strcasecmp(encoding_type.c_str(), "url") == 0) {
 797     s->formatter->dump_string("EncodingType", "url");
 798     encode_key = true;
 799   }
 800
 801   if (op_ret >= 0) {
 802     vector<rgw_bucket_dir_entry>::iterator iter;
 803     for (iter = objs.begin(); iter != objs.end(); ++iter) {
 804       rgw_obj_key key(iter->key);
 805       s->formatter->open_array_section("Contents");
 806       if (encode_key) {
 807         string key_name;
 808         url_encode(key.name, key_name);
 809         s->formatter->dump_string("Key", key_name);
 810       } else {
 811         s->formatter->dump_string("Key", key.name);
 812       }
 813       dump_time(s, "LastModified", &iter->meta.mtime);
 814       s->formatter->dump_format("ETag", "\"%s\"", iter->meta.etag.c_str());
 815       s->formatter->dump_int("Size", iter->meta.accounted_size);
 816       s->formatter->dump_string("StorageClass", "STANDARD");
 817       dump_owner(s, iter->meta.owner, iter->meta.owner_display_name);
 818       if (s->system_request) {
 819         s->formatter->dump_string("RgwxTag", iter->tag);
 820       }
 821       s->formatter->close_section();
 822     }
 823     if (!common_prefixes.empty()) {
 824       map<string, bool>::iterator pref_iter;
 825       for (pref_iter = common_prefixes.begin();
 826            pref_iter != common_prefixes.end(); ++pref_iter) {
 827         s->formatter->open_array_section("CommonPrefixes");
 828         s->formatter->dump_string("Prefix", pref_iter->first);
 829         s->formatter->close_section();
 830       }
 831     }
 832   }
 833   s->formatter->close_section();
 834   rgw_flush_formatter_and_reset(s, s->formatter);
 835 }
 836
 837 void RGWGetBucketLogging_ObjStore_S3::send_response()
 838 {
 839   dump_errno(s);
 840   end_header(s, this, "application/xml");
 841   dump_start(s);
 842
 843   s->formatter->open_object_section_in_ns("BucketLoggingStatus", XMLNS_AWS_S3);
 844   s->formatter->close_section();
 845   rgw_flush_formatter_and_reset(s, s->formatter);
 846 }
 847
 848 void RGWGetBucketLocation_ObjStore_S3::send_response()
 849 {
 850   dump_errno(s);
 851   end_header(s, this);
 852   dump_start(s);
 853
 854   RGWZoneGroup zonegroup;
 855   string api_name;
 856
 857   int ret = store->get_zonegroup(s->bucket_info.zonegroup, zonegroup);
 858   if (ret >= 0) {
 859     api_name = zonegroup.api_name;
 860   } else  {
 861     if (s->bucket_info.zonegroup != "default") {
 862       api_name = s->bucket_info.zonegroup;
 863     }
 864   }
 865
 866   s->formatter->dump_format_ns("LocationConstraint", XMLNS_AWS_S3,
 867                                "%s", api_name.c_str());
 868   rgw_flush_formatter_and_reset(s, s->formatter);
 869 }
 870
 871 void RGWGetBucketVersioning_ObjStore_S3::send_response()
 872 {
 873   dump_errno(s);
 874   end_header(s, this, "application/xml");
 875   dump_start(s);
 876
 877   s->formatter->open_object_section_in_ns("VersioningConfiguration", XMLNS_AWS_S3);
 878   if (versioned) {
 879     const char *status = (versioning_enabled ? "Enabled" : "Suspended");
 880     s->formatter->dump_string("Status", status);
 881   }
 882   s->formatter->close_section();
 883   rgw_flush_formatter_and_reset(s, s->formatter);
 884 }
 885
 886 class RGWSetBucketVersioningParser : public RGWXMLParser
 887 {
 888   XMLObj *alloc_obj(const char *el) override {
 889     return new XMLObj;
 890   }
 891
 892 public:
 893   RGWSetBucketVersioningParser() {}
 894   ~RGWSetBucketVersioningParser() override {}
 895
 896   int get_versioning_status(bool *status) {
 897     XMLObj *config = find_first("VersioningConfiguration");
 898     if (!config)
 899       return -EINVAL;
 900
 901     *status = false;
 902
 903     XMLObj *field = config->find_first("Status");
 904     if (!field)
 905       return 0;
 906
 907     string& s = field->get_data();
 908
 909     if (stringcasecmp(s, "Enabled") == 0) {
 910       *status = true;
 911     } else if (stringcasecmp(s, "Suspended") != 0) {
 912       return -EINVAL;
 913     }
 914
 915     return 0;
 916   }
 917 };
 918
 919 int RGWSetBucketVersioning_ObjStore_S3::get_params()
 920 {
 921   char *data = nullptr;
 922   int len = 0;
 923   int r =
 924     rgw_rest_read_all_input(s, &data, &len, s->cct->_conf->rgw_max_put_param_size, false);
 925   if (r < 0) {
 926     return r;
 927   }
 928
 929   auto data_deleter = std::unique_ptr<char, decltype(free)*>{data, free};
 930
 931   r = do_aws4_auth_completion();
 932   if (r < 0) {
 933     return r;
 934   }
 935
 936   RGWSetBucketVersioningParser parser;
 937
 938   if (!parser.init()) {
 939     ldout(s->cct, 0) << "ERROR: failed to initialize parser" << dendl;
 940     r = -EIO;
 941     return r;
 942   }
 943
 944   if (!parser.parse(data, len, 1)) {
 945     ldout(s->cct, 10) << "failed to parse data: " << data << dendl;
 946     r = -EINVAL;
 947     return r;
 948   }
 949
 950   if (!store->is_meta_master()) {
 951     /* only need to keep this data around if we're not meta master */
 952     in_data.append(data, len);
 953   }
 954
 955   r = parser.get_versioning_status(&enable_versioning);
 956
 957   return r;
 958 }
 959
 960 void RGWSetBucketVersioning_ObjStore_S3::send_response()
 961 {
 962   if (op_ret)
 963     set_req_state_err(s, op_ret);
 964   dump_errno(s);
 965   end_header(s);
 966 }
 967
 968 int RGWSetBucketWebsite_ObjStore_S3::get_params()
 969 {
 970   char *data = nullptr;
 971   int len = 0;
 972   const auto max_size = s->cct->_conf->rgw_max_put_param_size;
 973   int r = rgw_rest_read_all_input(s, &data, &len, max_size, false);
 974
 975   if (r < 0) {
 976     return r;
 977   }
 978
 979   auto data_deleter = std::unique_ptr<char, decltype(free)*>{data, free};
 980
 981   r = do_aws4_auth_completion();
 982   if (r < 0) {
 983     return r;
 984   }
 985
 986   bufferptr in_ptr(data, len);
 987   in_data.append(in_ptr);
 988
 989   RGWXMLDecoder::XMLParser parser;
 990   if (!parser.init()) {
 991     ldout(s->cct, 0) << "ERROR: failed to initialize parser" << dendl;
 992     return -EIO;
 993   }
 994
 995   if (!parser.parse(data, len, 1)) {
 996     string str(data, len);
 997     ldout(s->cct, 5) << "failed to parse xml: " << str << dendl;
 998     return -EINVAL;
 999   }
1000
1001   try {
1002     RGWXMLDecoder::decode_xml("WebsiteConfiguration", website_conf, &parser, true);
1003   } catch (RGWXMLDecoder::err& err) {
1004     string str(data, len);
1005     ldout(s->cct, 5) << "unexpected xml: " << str << dendl;
1006     return -EINVAL;
1007   }
1008
1009   return 0;
1010 }
1011
1012 void RGWSetBucketWebsite_ObjStore_S3::send_response()
1013 {
1014   if (op_ret < 0)
1015     set_req_state_err(s, op_ret);
1016   dump_errno(s);
1017   end_header(s);
1018 }
1019
1020 void RGWDeleteBucketWebsite_ObjStore_S3::send_response()
1021 {
1022   if (op_ret == 0) {
1023     op_ret = STATUS_NO_CONTENT;
1024   }
1025   set_req_state_err(s, op_ret);
1026   dump_errno(s);
1027   end_header(s);
1028 }
1029
1030 void RGWGetBucketWebsite_ObjStore_S3::send_response()
1031 {
1032   if (op_ret)
1033     set_req_state_err(s, op_ret);
1034   dump_errno(s);
1035   end_header(s, this, "application/xml");
1036   dump_start(s);
1037
1038   if (op_ret < 0) {
1039     return;
1040   }
1041
1042   RGWBucketWebsiteConf& conf = s->bucket_info.website_conf;
1043
1044   s->formatter->open_object_section_in_ns("WebsiteConfiguration", XMLNS_AWS_S3);
1045   conf.dump_xml(s->formatter);
1046   s->formatter->close_section(); // WebsiteConfiguration
1047   rgw_flush_formatter_and_reset(s, s->formatter);
1048 }
1049
1050 static void dump_bucket_metadata(struct req_state *s, RGWBucketEnt& bucket)
1051 {
1052   dump_header(s, "X-RGW-Object-Count", static_cast<long long>(bucket.count));
1053   dump_header(s, "X-RGW-Bytes-Used", static_cast<long long>(bucket.size));
1054 }
1055
1056 void RGWStatBucket_ObjStore_S3::send_response()
1057 {
1058   if (op_ret >= 0) {
1059     dump_bucket_metadata(s, bucket);
1060   }
1061
1062   set_req_state_err(s, op_ret);
1063   dump_errno(s);
1064
1065   end_header(s, this);
1066   dump_start(s);
1067 }
1068
1069 static int create_s3_policy(struct req_state *s, RGWRados *store,
1070                             RGWAccessControlPolicy_S3& s3policy,
1071                             ACLOwner& owner)
1072 {
1073   if (s->has_acl_header) {
1074     if (!s->canned_acl.empty())
1075       return -ERR_INVALID_REQUEST;
1076
1077     return s3policy.create_from_headers(store, s->info.env, owner);
1078   }
1079
1080   return s3policy.create_canned(owner, s->bucket_owner, s->canned_acl);
1081 }
1082
1083 class RGWLocationConstraint : public XMLObj
1084 {
1085 public:
1086   RGWLocationConstraint() {}
1087   ~RGWLocationConstraint() override {}
1088   bool xml_end(const char *el) override {
1089     if (!el)
1090       return false;
1091
1092     location_constraint = get_data();
1093
1094     return true;
1095   }
1096
1097   string location_constraint;
1098 };
1099
1100 class RGWCreateBucketConfig : public XMLObj
1101 {
1102 public:
1103   RGWCreateBucketConfig() {}
1104   ~RGWCreateBucketConfig() override {}
1105 };
1106
1107 class RGWCreateBucketParser : public RGWXMLParser
1108 {
1109   XMLObj *alloc_obj(const char *el) override {
1110     return new XMLObj;
1111   }
1112
1113 public:
1114   RGWCreateBucketParser() {}
1115   ~RGWCreateBucketParser() override {}
1116
1117   bool get_location_constraint(string& zone_group) {
1118     XMLObj *config = find_first("CreateBucketConfiguration");
1119     if (!config)
1120       return false;
1121
1122     XMLObj *constraint = config->find_first("LocationConstraint");
1123     if (!constraint)
1124       return false;
1125
1126     zone_group = constraint->get_data();
1127
1128     return true;
1129   }
1130 };
1131
1132 int RGWCreateBucket_ObjStore_S3::get_params()
1133 {
1134   RGWAccessControlPolicy_S3 s3policy(s->cct);
1135
1136   int r = create_s3_policy(s, store, s3policy, s->owner);
1137   if (r < 0)
1138     return r;
1139
1140   policy = s3policy;
1141
1142   int len = 0;
1143   char *data = nullptr;
1144
1145   const auto max_size = s->cct->_conf->rgw_max_put_param_size;
1146   op_ret = rgw_rest_read_all_input(s, &data, &len, max_size, false);
1147
1148   if ((op_ret < 0) && (op_ret != -ERR_LENGTH_REQUIRED))
1149     return op_ret;
1150
1151   auto data_deleter = std::unique_ptr<char, decltype(free)*>{data, free};
1152
1153   const int auth_ret = do_aws4_auth_completion();
1154   if (auth_ret < 0) {
1155     return auth_ret;
1156   }
1157
1158   bufferptr in_ptr(data, len);
1159   in_data.append(in_ptr);
1160
1161   if (len) {
1162     RGWCreateBucketParser parser;
1163
1164     if (!parser.init()) {
1165       ldout(s->cct, 0) << "ERROR: failed to initialize parser" << dendl;
1166       return -EIO;
1167     }
1168
1169     bool success = parser.parse(data, len, 1);
1170     ldout(s->cct, 20) << "create bucket input data=" << data << dendl;
1171
1172     if (!success) {
1173       ldout(s->cct, 0) << "failed to parse input: " << data << dendl;
1174       return -EINVAL;
1175     }
1176
1177     if (!parser.get_location_constraint(location_constraint)) {
1178       ldout(s->cct, 0) << "provided input did not specify location constraint correctly" << dendl;
1179       return -EINVAL;
1180     }
1181
1182     ldout(s->cct, 10) << "create bucket location constraint: "
1183                       << location_constraint << dendl;
1184   }
1185
1186   size_t pos = location_constraint.find(':');
1187   if (pos != string::npos) {
1188     placement_rule = location_constraint.substr(pos + 1);
1189     location_constraint = location_constraint.substr(0, pos);
1190   }
1191
1192   return 0;
1193 }
1194
1195 void RGWCreateBucket_ObjStore_S3::send_response()
1196 {
1197   if (op_ret == -ERR_BUCKET_EXISTS)
1198     op_ret = 0;
1199   if (op_ret)
1200     set_req_state_err(s, op_ret);
1201   dump_errno(s);
1202   end_header(s);
1203
1204   if (op_ret < 0)
1205     return;
1206
1207   if (s->system_request) {
1208     JSONFormatter f; /* use json formatter for system requests output */
1209
1210     f.open_object_section("info");
1211     encode_json("entry_point_object_ver", ep_objv, &f);
1212     encode_json("object_ver", info.objv_tracker.read_version, &f);
1213     encode_json("bucket_info", info, &f);
1214     f.close_section();
1215     rgw_flush_formatter_and_reset(s, &f);
1216   }
1217 }
1218
1219 void RGWDeleteBucket_ObjStore_S3::send_response()
1220 {
1221   int r = op_ret;
1222   if (!r)
1223     r = STATUS_NO_CONTENT;
1224
1225   set_req_state_err(s, r);
1226   dump_errno(s);
1227   end_header(s, this);
1228
1229   if (s->system_request) {
1230     JSONFormatter f; /* use json formatter for system requests output */
1231
1232     f.open_object_section("info");
1233     encode_json("object_ver", objv_tracker.read_version, &f);
1234     f.close_section();
1235     rgw_flush_formatter_and_reset(s, &f);
1236   }
1237 }
1238
1239 int RGWPutObj_ObjStore_S3::get_params()
1240 {
1241   if (!s->length)
1242     return -ERR_LENGTH_REQUIRED;
1243
1244   RGWObjectCtx& obj_ctx = *static_cast<RGWObjectCtx *>(s->obj_ctx);
1245   map<string, bufferlist> src_attrs;
1246   size_t pos;
1247   int ret;
1248
1249   RGWAccessControlPolicy_S3 s3policy(s->cct);
1250   ret = create_s3_policy(s, store, s3policy, s->owner);
1251   if (ret < 0)
1252     return ret;
1253
1254   policy = s3policy;
1255
1256   if_match = s->info.env->get("HTTP_IF_MATCH");
1257   if_nomatch = s->info.env->get("HTTP_IF_NONE_MATCH");
1258   copy_source = url_decode(s->info.env->get("HTTP_X_AMZ_COPY_SOURCE", ""));
1259   copy_source_range = s->info.env->get("HTTP_X_AMZ_COPY_SOURCE_RANGE");
1260
1261   /* handle x-amz-copy-source */
1262   boost::string_view cs_view(copy_source);
1263   if (! cs_view.empty()) {
1264     if (cs_view[0] == '/')
1265       cs_view.remove_prefix(1);
1266     copy_source_bucket_name = cs_view.to_string();
1267     pos = copy_source_bucket_name.find("/");
1268     if (pos == std::string::npos) {
1269       ret = -EINVAL;
1270       ldout(s->cct, 5) << "x-amz-copy-source bad format" << dendl;
1271       return ret;
1272     }
1273     copy_source_object_name =
1274       copy_source_bucket_name.substr(pos + 1, copy_source_bucket_name.size());
1275     copy_source_bucket_name = copy_source_bucket_name.substr(0, pos);
1276 #define VERSION_ID_STR "?versionId="
1277     pos = copy_source_object_name.find(VERSION_ID_STR);
1278     if (pos == std::string::npos) {
1279       copy_source_object_name = url_decode(copy_source_object_name);
1280     } else {
1281       copy_source_version_id =
1282         copy_source_object_name.substr(pos + sizeof(VERSION_ID_STR) - 1);
1283       copy_source_object_name =
1284         url_decode(copy_source_object_name.substr(0, pos));
1285     }
1286     pos = copy_source_bucket_name.find(":");
1287     if (pos == std::string::npos) {
1288        copy_source_tenant_name = s->src_tenant_name;
1289     } else {
1290        copy_source_tenant_name = copy_source_bucket_name.substr(0, pos);
1291        copy_source_bucket_name = copy_source_bucket_name.substr(pos + 1, copy_source_bucket_name.size());
1292        if (copy_source_bucket_name.empty()) {
1293          ret = -EINVAL;
1294          ldout(s->cct, 5) << "source bucket name is empty" << dendl;
1295          return ret;
1296        }
1297     }
1298     ret = store->get_bucket_info(obj_ctx,
1299                                  copy_source_tenant_name,
1300                                  copy_source_bucket_name,
1301                                  copy_source_bucket_info,
1302                                  NULL, &src_attrs);
1303     if (ret < 0) {
1304       ldout(s->cct, 5) << __func__ << "(): get_bucket_info() returned ret=" << ret << dendl;
1305       return ret;
1306     }
1307
1308     /* handle x-amz-copy-source-range */
1309
1310     if (copy_source_range) {
1311       string range = copy_source_range;
1312       pos = range.find("=");
1313       if (pos == std::string::npos) {
1314         ret = -EINVAL;
1315         ldout(s->cct, 5) << "x-amz-copy-source-range bad format" << dendl;
1316         return ret;
1317       }
1318       range = range.substr(pos + 1);
1319       pos = range.find("-");
1320       if (pos == std::string::npos) {
1321         ret = -EINVAL;
1322         ldout(s->cct, 5) << "x-amz-copy-source-range bad format" << dendl;
1323         return ret;
1324       }
1325       string first = range.substr(0, pos);
1326       string last = range.substr(pos + 1);
1327       copy_source_range_fst = strtoull(first.c_str(), NULL, 10);
1328       copy_source_range_lst = strtoull(last.c_str(), NULL, 10);
1329     }
1330
1331   } /* copy_source */
1332
1333   /* handle object tagging */
1334   auto tag_str = s->info.env->get("HTTP_X_AMZ_TAGGING");
1335   if (tag_str){
1336     obj_tags = ceph::make_unique<RGWObjTags>();
1337     ret = obj_tags->set_from_string(tag_str);
1338     if (ret < 0){
1339       ldout(s->cct,0) << "setting obj tags failed with " << ret << dendl;
1340       if (ret == -ERR_INVALID_TAG){
1341         ret = -EINVAL; //s3 returns only -EINVAL for PUT requests
1342       }
1343
1344       return ret;
1345     }
1346   }
1347
1348   return RGWPutObj_ObjStore::get_params();
1349 }
1350
1351 int RGWPutObj_ObjStore_S3::get_data(bufferlist& bl)
1352 {
1353   const int ret = RGWPutObj_ObjStore::get_data(bl);
1354   if (ret == 0) {
1355     const int ret_auth = do_aws4_auth_completion();
1356     if (ret_auth < 0) {
1357       return ret_auth;
1358     }
1359   }
1360
1361   return ret;
1362 }
1363
1364 static int get_success_retcode(int code)
1365 {
1366   switch (code) {
1367     case 201:
1368       return STATUS_CREATED;
1369     case 204:
1370       return STATUS_NO_CONTENT;
1371   }
1372   return 0;
1373 }
1374
1375 void RGWPutObj_ObjStore_S3::send_response()
1376 {
1377   if (op_ret) {
1378     set_req_state_err(s, op_ret);
1379     dump_errno(s);
1380   } else {
1381     if (s->cct->_conf->rgw_s3_success_create_obj_status) {
1382       op_ret = get_success_retcode(
1383         s->cct->_conf->rgw_s3_success_create_obj_status);
1384       set_req_state_err(s, op_ret);
1385     }
1386     if (copy_source.empty()) {
1387       dump_errno(s);
1388       dump_etag(s, etag);
1389       dump_content_length(s, 0);
1390       for (auto &it : crypt_http_responses)
1391         dump_header(s, it.first, it.second);
1392     } else {
1393       dump_errno(s);
1394       end_header(s, this, "application/xml");
1395       dump_start(s);
1396       struct tm tmp;
1397       utime_t ut(mtime);
1398       time_t secs = (time_t)ut.sec();
1399       gmtime_r(&secs, &tmp);
1400       char buf[TIME_BUF_SIZE];
1401       s->formatter->open_object_section_in_ns("CopyPartResult",
1402           "http://s3.amazonaws.com/doc/2006-03-01/");
1403       if (strftime(buf, sizeof(buf), "%Y-%m-%dT%T.000Z", &tmp) > 0) {
1404         s->formatter->dump_string("LastModified", buf);
1405       }
1406       s->formatter->dump_string("ETag", etag);
1407       s->formatter->close_section();
1408       rgw_flush_formatter_and_reset(s, s->formatter);
1409       return;
1410     }
1411   }
1412   if (s->system_request && !real_clock::is_zero(mtime)) {
1413     dump_epoch_header(s, "Rgwx-Mtime", mtime);
1414   }
1415   end_header(s, this);
1416 }
1417
1418 static inline int get_obj_attrs(RGWRados *store, struct req_state *s, rgw_obj& obj, map<string, bufferlist>& attrs)
1419 {
1420   RGWRados::Object op_target(store, s->bucket_info, *static_cast<RGWObjectCtx *>(s->obj_ctx), obj);
1421   RGWRados::Object::Read read_op(&op_target);
1422
1423   read_op.params.attrs = &attrs;
1424
1425   return read_op.prepare();
1426 }
1427
1428 static inline void set_attr(map<string, bufferlist>& attrs, const char* key, const std::string& value)
1429 {
1430   bufferlist bl;
1431   ::encode(value,bl);
1432   attrs.emplace(key, std::move(bl));
1433 }
1434
1435 static inline void set_attr(map<string, bufferlist>& attrs, const char* key, const char* value)
1436 {
1437   bufferlist bl;
1438   ::encode(value,bl);
1439   attrs.emplace(key, std::move(bl));
1440 }
1441
1442 int RGWPutObj_ObjStore_S3::get_decrypt_filter(
1443     std::unique_ptr<RGWGetDataCB>* filter,
1444     RGWGetDataCB* cb,
1445     map<string, bufferlist>& attrs,
1446     bufferlist* manifest_bl)
1447 {
1448   std::map<std::string, std::string> crypt_http_responses_unused;
1449
1450   int res = 0;
1451   std::unique_ptr<BlockCrypt> block_crypt;
1452   res = rgw_s3_prepare_decrypt(s, attrs, &block_crypt, crypt_http_responses_unused);
1453   if (res == 0) {
1454     if (block_crypt != nullptr) {
1455       auto f = std::unique_ptr<RGWGetObj_BlockDecrypt>(new RGWGetObj_BlockDecrypt(s->cct, cb, std::move(block_crypt)));
1456       //RGWGetObj_BlockDecrypt* f = new RGWGetObj_BlockDecrypt(s->cct, cb, std::move(block_crypt));
1457       if (f != nullptr) {
1458         if (manifest_bl != nullptr) {
1459           res = f->read_manifest(*manifest_bl);
1460           if (res == 0) {
1461             *filter = std::move(f);
1462           }
1463         }
1464       }
1465     }
1466   }
1467   return res;
1468 }
1469
1470 int RGWPutObj_ObjStore_S3::get_encrypt_filter(
1471     std::unique_ptr<RGWPutObjDataProcessor>* filter,
1472     RGWPutObjDataProcessor* cb)
1473 {
1474   int res = 0;
1475   RGWPutObjProcessor_Multipart* multi_processor=dynamic_cast<RGWPutObjProcessor_Multipart*>(cb);
1476   if (multi_processor != nullptr) {
1477     RGWMPObj* mp = nullptr;
1478     multi_processor->get_mp(&mp);
1479     if (mp != nullptr) {
1480       map<string, bufferlist> xattrs;
1481       string meta_oid;
1482       meta_oid = mp->get_meta();
1483
1484       rgw_obj obj;
1485       obj.init_ns(s->bucket, meta_oid, RGW_OBJ_NS_MULTIPART);
1486       obj.set_in_extra_data(true);
1487       res = get_obj_attrs(store, s, obj, xattrs);
1488       if (res == 0) {
1489         std::unique_ptr<BlockCrypt> block_crypt;
1490         /* We are adding to existing object.
1491          * We use crypto mode that configured as if we were decrypting. */
1492         res = rgw_s3_prepare_decrypt(s, xattrs, &block_crypt, crypt_http_responses);
1493         if (res == 0 && block_crypt != nullptr)
1494           *filter = std::unique_ptr<RGWPutObj_BlockEncrypt>(
1495               new RGWPutObj_BlockEncrypt(s->cct, cb, std::move(block_crypt)));
1496       }
1497     }
1498     /* it is ok, to not have encryption at all */
1499   }
1500   else
1501   {
1502     std::unique_ptr<BlockCrypt> block_crypt;
1503     res = rgw_s3_prepare_encrypt(s, attrs, nullptr, &block_crypt, crypt_http_responses);
1504     if (res == 0 && block_crypt != nullptr) {
1505       *filter = std::unique_ptr<RGWPutObj_BlockEncrypt>(
1506           new RGWPutObj_BlockEncrypt(s->cct, cb, std::move(block_crypt)));
1507     }
1508   }
1509   return res;
1510 }
1511
1512 void RGWPostObj_ObjStore_S3::rebuild_key(string& key)
1513 {
1514   static string var = "${filename}";
1515   int pos = key.find(var);
1516   if (pos < 0)
1517     return;
1518
1519   string new_key = key.substr(0, pos);
1520   new_key.append(filename);
1521   new_key.append(key.substr(pos + var.size()));
1522
1523   key = new_key;
1524 }
1525
1526 std::string RGWPostObj_ObjStore_S3::get_current_filename() const
1527 {
1528   return s->object.name;
1529 }
1530
1531 std::string RGWPostObj_ObjStore_S3::get_current_content_type() const
1532 {
1533   return content_type;
1534 }
1535
1536 int RGWPostObj_ObjStore_S3::get_params()
1537 {
1538   op_ret = RGWPostObj_ObjStore::get_params();
1539   if (op_ret < 0) {
1540     return op_ret;
1541   }
1542
1543   ldout(s->cct, 20) << "adding bucket to policy env: " << s->bucket.name
1544                     << dendl;
1545   env.add_var("bucket", s->bucket.name);
1546
1547   bool done;
1548   do {
1549     struct post_form_part part;
1550     int r = read_form_part_header(&part, done);
1551     if (r < 0)
1552       return r;
1553
1554     if (s->cct->_conf->subsys.should_gather(ceph_subsys_rgw, 20)) {
1555       ldout(s->cct, 20) << "read part header -- part.name="
1556                         << part.name << dendl;
1557
1558       for (const auto& pair : part.fields) {
1559         ldout(s->cct, 20) << "field.name=" << pair.first << dendl;
1560         ldout(s->cct, 20) << "field.val=" << pair.second.val << dendl;
1561         ldout(s->cct, 20) << "field.params:" << dendl;
1562
1563         for (const auto& param_pair : pair.second.params) {
1564           ldout(s->cct, 20) << " " << param_pair.first
1565                             << " -> " << param_pair.second << dendl;
1566         }
1567       }
1568     }
1569
1570     if (done) { /* unexpected here */
1571       err_msg = "Malformed request";
1572       return -EINVAL;
1573     }
1574
1575     if (stringcasecmp(part.name, "file") == 0) { /* beginning of data transfer */
1576       struct post_part_field& field = part.fields["Content-Disposition"];
1577       map<string, string>::iterator iter = field.params.find("filename");
1578       if (iter != field.params.end()) {
1579         filename = iter->second;
1580       }
1581       parts[part.name] = part;
1582       break;
1583     }
1584
1585     bool boundary;
1586     uint64_t chunk_size = s->cct->_conf->rgw_max_chunk_size;
1587     r = read_data(part.data, chunk_size, boundary, done);
1588     if (r < 0 || !boundary) {
1589       err_msg = "Couldn't find boundary";
1590       return -EINVAL;
1591     }
1592     parts[part.name] = part;
1593     string part_str(part.data.c_str(), part.data.length());
1594     env.add_var(part.name, part_str);
1595   } while (!done);
1596
1597   string object_str;
1598   if (!part_str(parts, "key", &object_str)) {
1599     err_msg = "Key not specified";
1600     return -EINVAL;
1601   }
1602
1603   s->object = rgw_obj_key(object_str);
1604
1605   rebuild_key(s->object.name);
1606
1607   if (s->object.empty()) {
1608     err_msg = "Empty object name";
1609     return -EINVAL;
1610   }
1611
1612   env.add_var("key", s->object.name);
1613
1614   part_str(parts, "Content-Type", &content_type);
1615
1616   /* AWS permits POST without Content-Type: http://tracker.ceph.com/issues/20201 */
1617   if (! content_type.empty()) {
1618     env.add_var("Content-Type", content_type);
1619   }
1620
1621   map<string, struct post_form_part, ltstr_nocase>::iterator piter =
1622     parts.upper_bound(RGW_AMZ_META_PREFIX);
1623   for (; piter != parts.end(); ++piter) {
1624     string n = piter->first;
1625     if (strncasecmp(n.c_str(), RGW_AMZ_META_PREFIX,
1626                     sizeof(RGW_AMZ_META_PREFIX) - 1) != 0)
1627       break;
1628
1629     string attr_name = RGW_ATTR_PREFIX;
1630     attr_name.append(n);
1631
1632     /* need to null terminate it */
1633     bufferlist& data = piter->second.data;
1634     string str = string(data.c_str(), data.length());
1635
1636     bufferlist attr_bl;
1637     attr_bl.append(str.c_str(), str.size() + 1);
1638
1639     attrs[attr_name] = attr_bl;
1640   }
1641   // TODO: refactor this and the above loop to share code
1642   piter = parts.find(RGW_AMZ_WEBSITE_REDIRECT_LOCATION);
1643   if (piter != parts.end()) {
1644     string n = piter->first;
1645     string attr_name = RGW_ATTR_PREFIX;
1646     attr_name.append(n);
1647     /* need to null terminate it */
1648     bufferlist& data = piter->second.data;
1649     string str = string(data.c_str(), data.length());
1650
1651     bufferlist attr_bl;
1652     attr_bl.append(str.c_str(), str.size() + 1);
1653
1654     attrs[attr_name] = attr_bl;
1655   }
1656
1657   int r = get_policy();
1658   if (r < 0)
1659     return r;
1660
1661   r = get_tags();
1662   if (r < 0)
1663     return r;
1664
1665
1666   min_len = post_policy.min_length;
1667   max_len = post_policy.max_length;
1668
1669
1670
1671   return 0;
1672 }
1673
1674 int RGWPostObj_ObjStore_S3::get_tags()
1675 {
1676   string tags_str;
1677   if (part_str(parts, "tagging", &tags_str)) {
1678     RGWObjTagsXMLParser parser;
1679     if (!parser.init()){
1680       ldout(s->cct, 0) << "Couldn't init RGWObjTags XML parser" << dendl;
1681       err_msg = "Server couldn't process the request";
1682       return -EINVAL; // TODO: This class of errors in rgw code should be a 5XX error
1683     }
1684     if (!parser.parse(tags_str.c_str(), tags_str.size(), 1)) {
1685       ldout(s->cct,0 ) << "Invalid Tagging XML" << dendl;
1686       err_msg = "Invalid Tagging XML";
1687       return -EINVAL;
1688     }
1689
1690     RGWObjTagSet_S3 *obj_tags_s3;
1691     RGWObjTagging_S3 *tagging;
1692
1693     tagging = static_cast<RGWObjTagging_S3 *>(parser.find_first("Tagging"));
1694     obj_tags_s3 = static_cast<RGWObjTagSet_S3 *>(tagging->find_first("TagSet"));
1695     if(!obj_tags_s3){
1696       return -ERR_MALFORMED_XML;
1697     }
1698
1699     RGWObjTags obj_tags;
1700     int r = obj_tags_s3->rebuild(obj_tags);
1701     if (r < 0)
1702       return r;
1703
1704     bufferlist tags_bl;
1705     obj_tags.encode(tags_bl);
1706     ldout(s->cct, 20) << "Read " << obj_tags.count() << "tags" << dendl;
1707     attrs[RGW_ATTR_TAGS] = tags_bl;
1708   }
1709
1710
1711   return 0;
1712 }
1713
1714 int RGWPostObj_ObjStore_S3::get_policy()
1715 {
1716   if (part_bl(parts, "policy", &s->auth.s3_postobj_creds.encoded_policy)) {
1717     bool aws4_auth = false;
1718
1719     /* x-amz-algorithm handling */
1720     using rgw::auth::s3::AWS4_HMAC_SHA256_STR;
1721     if ((part_str(parts, "x-amz-algorithm", &s->auth.s3_postobj_creds.x_amz_algorithm)) &&
1722         (s->auth.s3_postobj_creds.x_amz_algorithm == AWS4_HMAC_SHA256_STR)) {
1723       ldout(s->cct, 0) << "Signature verification algorithm AWS v4 (AWS4-HMAC-SHA256)" << dendl;
1724       aws4_auth = true;
1725     } else {
1726       ldout(s->cct, 0) << "Signature verification algorithm AWS v2" << dendl;
1727     }
1728
1729     // check that the signature matches the encoded policy
1730     if (aws4_auth) {
1731       /* AWS4 */
1732
1733       /* x-amz-credential handling */
1734       if (!part_str(parts, "x-amz-credential",
1735                     &s->auth.s3_postobj_creds.x_amz_credential)) {
1736         ldout(s->cct, 0) << "No S3 aws4 credential found!" << dendl;
1737         err_msg = "Missing aws4 credential";
1738         return -EINVAL;
1739       }
1740
1741       /* x-amz-signature handling */
1742       if (!part_str(parts, "x-amz-signature",
1743                     &s->auth.s3_postobj_creds.signature)) {
1744         ldout(s->cct, 0) << "No aws4 signature found!" << dendl;
1745         err_msg = "Missing aws4 signature";
1746         return -EINVAL;
1747       }
1748
1749       /* x-amz-date handling */
1750       std::string received_date_str;
1751       if (!part_str(parts, "x-amz-date", &received_date_str)) {
1752         ldout(s->cct, 0) << "No aws4 date found!" << dendl;
1753         err_msg = "Missing aws4 date";
1754         return -EINVAL;
1755       }
1756     } else {
1757       /* AWS2 */
1758
1759       // check that the signature matches the encoded policy
1760       if (!part_str(parts, "AWSAccessKeyId",
1761                     &s->auth.s3_postobj_creds.access_key)) {
1762         ldout(s->cct, 0) << "No S3 aws2 access key found!" << dendl;
1763         err_msg = "Missing aws2 access key";
1764         return -EINVAL;
1765       }
1766
1767       if (!part_str(parts, "signature", &s->auth.s3_postobj_creds.signature)) {
1768         ldout(s->cct, 0) << "No aws2 signature found!" << dendl;
1769         err_msg = "Missing aws2 signature";
1770         return -EINVAL;
1771       }
1772     }
1773
1774     /* FIXME: this is a makeshift solution. The browser upload authentication will be
1775      * handled by an instance of rgw::auth::Completer spawned in Handler's authorize()
1776      * method. */
1777     const int ret = rgw::auth::Strategy::apply(auth_registry_ptr->get_s3_post(), s);
1778     if (ret != 0) {
1779       return -EACCES;
1780     } else {
1781       /* Populate the owner info. */
1782       s->owner.set_id(s->user->user_id);
1783       s->owner.set_name(s->user->display_name);
1784       ldout(s->cct, 20) << "Successful Signature Verification!" << dendl;
1785     }
1786
1787     ceph::bufferlist decoded_policy;
1788     try {
1789       decoded_policy.decode_base64(s->auth.s3_postobj_creds.encoded_policy);
1790     } catch (buffer::error& err) {
1791       ldout(s->cct, 0) << "failed to decode_base64 policy" << dendl;
1792       err_msg = "Could not decode policy";
1793       return -EINVAL;
1794     }
1795
1796     decoded_policy.append('\0'); // NULL terminate
1797     ldout(s->cct, 20) << "POST policy: " << decoded_policy.c_str() << dendl;
1798
1799
1800     int r = post_policy.from_json(decoded_policy, err_msg);
1801     if (r < 0) {
1802       if (err_msg.empty()) {
1803         err_msg = "Failed to parse policy";
1804       }
1805       ldout(s->cct, 0) << "failed to parse policy" << dendl;
1806       return -EINVAL;
1807     }
1808
1809     if (aws4_auth) {
1810       /* AWS4 */
1811       post_policy.set_var_checked("x-amz-signature");
1812     } else {
1813       /* AWS2 */
1814       post_policy.set_var_checked("AWSAccessKeyId");
1815       post_policy.set_var_checked("signature");
1816     }
1817     post_policy.set_var_checked("policy");
1818
1819     r = post_policy.check(&env, err_msg);
1820     if (r < 0) {
1821       if (err_msg.empty()) {
1822         err_msg = "Policy check failed";
1823       }
1824       ldout(s->cct, 0) << "policy check failed" << dendl;
1825       return r;
1826     }
1827
1828   } else {
1829     ldout(s->cct, 0) << "No attached policy found!" << dendl;
1830   }
1831
1832   string canned_acl;
1833   part_str(parts, "acl", &canned_acl);
1834
1835   RGWAccessControlPolicy_S3 s3policy(s->cct);
1836   ldout(s->cct, 20) << "canned_acl=" << canned_acl << dendl;
1837   if (s3policy.create_canned(s->owner, s->bucket_owner, canned_acl) < 0) {
1838     err_msg = "Bad canned ACLs";
1839     return -EINVAL;
1840   }
1841
1842   policy = s3policy;
1843
1844   return 0;
1845 }
1846
1847 int RGWPostObj_ObjStore_S3::complete_get_params()
1848 {
1849   bool done;
1850   do {
1851     struct post_form_part part;
1852     int r = read_form_part_header(&part, done);
1853     if (r < 0) {
1854       return r;
1855     }
1856
1857     ceph::bufferlist part_data;
1858     bool boundary;
1859     uint64_t chunk_size = s->cct->_conf->rgw_max_chunk_size;
1860     r = read_data(part.data, chunk_size, boundary, done);
1861     if (r < 0 || !boundary) {
1862       return -EINVAL;
1863     }
1864
1865     /* Just reading the data but not storing any results of that. */
1866   } while (!done);
1867
1868   return 0;
1869 }
1870
1871 int RGWPostObj_ObjStore_S3::get_data(ceph::bufferlist& bl, bool& again)
1872 {
1873   bool boundary;
1874   bool done;
1875
1876   const uint64_t chunk_size = s->cct->_conf->rgw_max_chunk_size;
1877   int r = read_data(bl, chunk_size, boundary, done);
1878   if (r < 0) {
1879     return r;
1880   }
1881
1882   if (boundary) {
1883     if (!done) {
1884       /* Reached end of data, let's drain the rest of the params */
1885       r = complete_get_params();
1886       if (r < 0) {
1887        return r;
1888       }
1889     }
1890   }
1891
1892   again = !boundary;
1893   return bl.length();
1894 }
1895
1896 void RGWPostObj_ObjStore_S3::send_response()
1897 {
1898   if (op_ret == 0 && parts.count("success_action_redirect")) {
1899     string redirect;
1900
1901     part_str(parts, "success_action_redirect", &redirect);
1902
1903     string tenant;
1904     string bucket;
1905     string key;
1906     string etag_str = "\"";
1907
1908     etag_str.append(etag);
1909     etag_str.append("\"");
1910
1911     string etag_url;
1912
1913     url_encode(s->bucket_tenant, tenant); /* surely overkill, but cheap */
1914     url_encode(s->bucket_name, bucket);
1915     url_encode(s->object.name, key);
1916     url_encode(etag_str, etag_url);
1917
1918     if (!s->bucket_tenant.empty()) {
1919       /*
1920        * What we really would like is to quaily the bucket name, so
1921        * that the client could simply copy it and paste into next request.
1922        * Unfortunately, in S3 we cannot know if the client will decide
1923        * to come through DNS, with "bucket.tenant" sytanx, or through
1924        * URL with "tenant\bucket" syntax. Therefore, we provide the
1925        * tenant separately.
1926        */
1927       redirect.append("?tenant=");
1928       redirect.append(tenant);
1929       redirect.append("&bucket=");
1930       redirect.append(bucket);
1931     } else {
1932       redirect.append("?bucket=");
1933       redirect.append(bucket);
1934     }
1935     redirect.append("&key=");
1936     redirect.append(key);
1937     redirect.append("&etag=");
1938     redirect.append(etag_url);
1939
1940     int r = check_utf8(redirect.c_str(), redirect.size());
1941     if (r < 0) {
1942       op_ret = r;
1943       goto done;
1944     }
1945     dump_redirect(s, redirect);
1946     op_ret = STATUS_REDIRECT;
1947   } else if (op_ret == 0 && parts.count("success_action_status")) {
1948     string status_string;
1949     uint32_t status_int;
1950
1951     part_str(parts, "success_action_status", &status_string);
1952
1953     int r = stringtoul(status_string, &status_int);
1954     if (r < 0) {
1955       op_ret = r;
1956       goto done;
1957     }
1958
1959     switch (status_int) {
1960       case 200:
1961         break;
1962       case 201:
1963         op_ret = STATUS_CREATED;
1964         break;
1965       default:
1966         op_ret = STATUS_NO_CONTENT;
1967         break;
1968     }
1969   } else if (! op_ret) {
1970     op_ret = STATUS_NO_CONTENT;
1971   }
1972
1973 done:
1974   if (op_ret == STATUS_CREATED) {
1975     for (auto &it : crypt_http_responses)
1976       dump_header(s, it.first, it.second);
1977     s->formatter->open_object_section("PostResponse");
1978     if (g_conf->rgw_dns_name.length())
1979       s->formatter->dump_format("Location", "%s/%s",
1980                                 s->info.script_uri.c_str(),
1981                                 s->object.name.c_str());
1982     if (!s->bucket_tenant.empty())
1983       s->formatter->dump_string("Tenant", s->bucket_tenant);
1984     s->formatter->dump_string("Bucket", s->bucket_name);
1985     s->formatter->dump_string("Key", s->object.name);
1986     s->formatter->close_section();
1987   }
1988   s->err.message = err_msg;
1989   set_req_state_err(s, op_ret);
1990   dump_errno(s);
1991   if (op_ret >= 0) {
1992     dump_content_length(s, s->formatter->get_len());
1993   }
1994   end_header(s, this);
1995   if (op_ret != STATUS_CREATED)
1996     return;
1997
1998   rgw_flush_formatter_and_reset(s, s->formatter);
1999 }
2000
2001 int RGWPostObj_ObjStore_S3::get_encrypt_filter(
2002     std::unique_ptr<RGWPutObjDataProcessor>* filter, RGWPutObjDataProcessor* cb)
2003 {
2004   int res = 0;
2005   std::unique_ptr<BlockCrypt> block_crypt;
2006   res = rgw_s3_prepare_encrypt(s, attrs, &parts, &block_crypt, crypt_http_responses);
2007   if (res == 0 && block_crypt != nullptr) {
2008     *filter = std::unique_ptr<RGWPutObj_BlockEncrypt>(
2009         new RGWPutObj_BlockEncrypt(s->cct, cb, std::move(block_crypt)));
2010   }
2011   else
2012     *filter = nullptr;
2013   return res;
2014 }
2015
2016 int RGWDeleteObj_ObjStore_S3::get_params()
2017 {
2018   const char *if_unmod = s->info.env->get("HTTP_X_AMZ_DELETE_IF_UNMODIFIED_SINCE");
2019
2020   if (s->system_request) {
2021     s->info.args.get_bool(RGW_SYS_PARAM_PREFIX "no-precondition-error", &no_precondition_error, false);
2022   }
2023
2024   if (if_unmod) {
2025     std::string if_unmod_decoded = url_decode(if_unmod);
2026     uint64_t epoch;
2027     uint64_t nsec;
2028     if (utime_t::parse_date(if_unmod_decoded, &epoch, &nsec) < 0) {
2029       ldout(s->cct, 10) << "failed to parse time: " << if_unmod_decoded << dendl;
2030       return -EINVAL;
2031     }
2032     unmod_since = utime_t(epoch, nsec).to_real_time();
2033   }
2034
2035   return 0;
2036 }
2037
2038 void RGWDeleteObj_ObjStore_S3::send_response()
2039 {
2040   int r = op_ret;
2041   if (r == -ENOENT)
2042     r = 0;
2043   if (!r)
2044     r = STATUS_NO_CONTENT;
2045
2046   set_req_state_err(s, r);
2047   dump_errno(s);
2048   if (!version_id.empty()) {
2049     dump_header(s, "x-amz-version-id", version_id);
2050   }
2051   if (delete_marker) {
2052     dump_header(s, "x-amz-delete-marker", "true");
2053   }
2054   end_header(s, this);
2055 }
2056
2057 int RGWCopyObj_ObjStore_S3::init_dest_policy()
2058 {
2059   RGWAccessControlPolicy_S3 s3policy(s->cct);
2060
2061   /* build a policy for the target object */
2062   int r = create_s3_policy(s, store, s3policy, s->owner);
2063   if (r < 0)
2064     return r;
2065
2066   dest_policy = s3policy;
2067
2068   return 0;
2069 }
2070
2071 int RGWCopyObj_ObjStore_S3::get_params()
2072 {
2073   if_mod = s->info.env->get("HTTP_X_AMZ_COPY_IF_MODIFIED_SINCE");
2074   if_unmod = s->info.env->get("HTTP_X_AMZ_COPY_IF_UNMODIFIED_SINCE");
2075   if_match = s->info.env->get("HTTP_X_AMZ_COPY_IF_MATCH");
2076   if_nomatch = s->info.env->get("HTTP_X_AMZ_COPY_IF_NONE_MATCH");
2077
2078   src_tenant_name = s->src_tenant_name;
2079   src_bucket_name = s->src_bucket_name;
2080   src_object = s->src_object;
2081   dest_tenant_name = s->bucket.tenant;
2082   dest_bucket_name = s->bucket.name;
2083   dest_object = s->object.name;
2084
2085   if (s->system_request) {
2086     source_zone = s->info.args.get(RGW_SYS_PARAM_PREFIX "source-zone");
2087     s->info.args.get_bool(RGW_SYS_PARAM_PREFIX "copy-if-newer", &copy_if_newer, false);
2088     if (!source_zone.empty()) {
2089       client_id = s->info.args.get(RGW_SYS_PARAM_PREFIX "client-id");
2090       op_id = s->info.args.get(RGW_SYS_PARAM_PREFIX "op-id");
2091
2092       if (client_id.empty() || op_id.empty()) {
2093         ldout(s->cct, 0) <<
2094           RGW_SYS_PARAM_PREFIX "client-id or "
2095           RGW_SYS_PARAM_PREFIX "op-id were not provided, "
2096           "required for intra-region copy"
2097                          << dendl;
2098         return -EINVAL;
2099       }
2100     }
2101   }
2102
2103   const char *md_directive = s->info.env->get("HTTP_X_AMZ_METADATA_DIRECTIVE");
2104   if (md_directive) {
2105     if (strcasecmp(md_directive, "COPY") == 0) {
2106       attrs_mod = RGWRados::ATTRSMOD_NONE;
2107     } else if (strcasecmp(md_directive, "REPLACE") == 0) {
2108       attrs_mod = RGWRados::ATTRSMOD_REPLACE;
2109     } else if (!source_zone.empty()) {
2110       attrs_mod = RGWRados::ATTRSMOD_NONE; // default for intra-zone_group copy
2111     } else {
2112       ldout(s->cct, 0) << "invalid metadata directive" << dendl;
2113       return -EINVAL;
2114     }
2115   }
2116
2117   if (source_zone.empty() &&
2118       (dest_tenant_name.compare(src_tenant_name) == 0) &&
2119       (dest_bucket_name.compare(src_bucket_name) == 0) &&
2120       (dest_object.compare(src_object.name) == 0) &&
2121       src_object.instance.empty() &&
2122       (attrs_mod != RGWRados::ATTRSMOD_REPLACE)) {
2123     /* can only copy object into itself if replacing attrs */
2124     ldout(s->cct, 0) << "can't copy object into itself if not replacing attrs"
2125                      << dendl;
2126     return -ERR_INVALID_REQUEST;
2127   }
2128   return 0;
2129 }
2130
2131 void RGWCopyObj_ObjStore_S3::send_partial_response(off_t ofs)
2132 {
2133   if (! sent_header) {
2134     if (op_ret)
2135     set_req_state_err(s, op_ret);
2136     dump_errno(s);
2137
2138     end_header(s, this, "application/xml");
2139     dump_start(s);
2140     if (op_ret == 0) {
2141       s->formatter->open_object_section_in_ns("CopyObjectResult", XMLNS_AWS_S3);
2142     }
2143     sent_header = true;
2144   } else {
2145     /* Send progress field. Note that this diverge from the original S3
2146      * spec. We do this in order to keep connection alive.
2147      */
2148     s->formatter->dump_int("Progress", (uint64_t)ofs);
2149   }
2150   rgw_flush_formatter(s, s->formatter);
2151 }
2152
2153 void RGWCopyObj_ObjStore_S3::send_response()
2154 {
2155   if (!sent_header)
2156     send_partial_response(0);
2157
2158   if (op_ret == 0) {
2159     dump_time(s, "LastModified", &mtime);
2160     std::string etag_str = etag.to_str();
2161     if (! etag_str.empty()) {
2162       s->formatter->dump_string("ETag", std::move(etag_str));
2163     }
2164     s->formatter->close_section();
2165     rgw_flush_formatter_and_reset(s, s->formatter);
2166   }
2167 }
2168
2169 void RGWGetACLs_ObjStore_S3::send_response()
2170 {
2171   if (op_ret)
2172     set_req_state_err(s, op_ret);
2173   dump_errno(s);
2174   end_header(s, this, "application/xml");
2175   dump_start(s);
2176   rgw_flush_formatter(s, s->formatter);
2177   dump_body(s, acls);
2178 }
2179
2180 int RGWPutACLs_ObjStore_S3::get_params()
2181 {
2182   int ret =  RGWPutACLs_ObjStore::get_params();
2183   if (ret >= 0) {
2184     const int ret_auth = do_aws4_auth_completion();
2185     if (ret_auth < 0) {
2186       return ret_auth;
2187     }
2188   }
2189   return ret;
2190 }
2191
2192 int RGWPutACLs_ObjStore_S3::get_policy_from_state(RGWRados *store,
2193                                                   struct req_state *s,
2194                                                   stringstream& ss)
2195 {
2196   RGWAccessControlPolicy_S3 s3policy(s->cct);
2197
2198   // bucket-* canned acls do not apply to bucket
2199   if (s->object.empty()) {
2200     if (s->canned_acl.find("bucket") != string::npos)
2201       s->canned_acl.clear();
2202   }
2203
2204   int r = create_s3_policy(s, store, s3policy, owner);
2205   if (r < 0)
2206     return r;
2207
2208   s3policy.to_xml(ss);
2209
2210   return 0;
2211 }
2212
2213 void RGWPutACLs_ObjStore_S3::send_response()
2214 {
2215   if (op_ret)
2216     set_req_state_err(s, op_ret);
2217   dump_errno(s);
2218   end_header(s, this, "application/xml");
2219   dump_start(s);
2220 }
2221
2222 void RGWGetLC_ObjStore_S3::execute()
2223 {
2224   config.set_ctx(s->cct);
2225
2226   map<string, bufferlist>::iterator aiter = s->bucket_attrs.find(RGW_ATTR_LC);
2227   if (aiter == s->bucket_attrs.end()) {
2228     op_ret = -ENOENT;
2229     return;
2230   }
2231
2232   bufferlist::iterator iter(&aiter->second);
2233   try {
2234       config.decode(iter);
2235     } catch (const buffer::error& e) {
2236       ldout(s->cct, 0) << __func__ <<  "decode life cycle config failed" << dendl;
2237       op_ret = -EIO;
2238       return;
2239     }
2240 }
2241
2242 void RGWGetLC_ObjStore_S3::send_response()
2243 {
2244   if (op_ret) {
2245     if (op_ret == -ENOENT) {
2246       set_req_state_err(s, ERR_NO_SUCH_LC);
2247     } else {
2248       set_req_state_err(s, op_ret);
2249     }
2250   }
2251   dump_errno(s);
2252   end_header(s, this, "application/xml");
2253   dump_start(s);
2254
2255   if (op_ret < 0)
2256     return;
2257
2258   config.dump_xml(s->formatter);
2259   rgw_flush_formatter_and_reset(s, s->formatter);
2260 }
2261
2262 void RGWPutLC_ObjStore_S3::send_response()
2263 {
2264   if (op_ret)
2265     set_req_state_err(s, op_ret);
2266   dump_errno(s);
2267   end_header(s, this, "application/xml");
2268   dump_start(s);
2269 }
2270
2271 void RGWDeleteLC_ObjStore_S3::send_response()
2272 {
2273   if (op_ret == 0)
2274       op_ret = STATUS_NO_CONTENT;
2275   if (op_ret) {
2276     set_req_state_err(s, op_ret);
2277   }
2278   dump_errno(s);
2279   end_header(s, this, "application/xml");
2280   dump_start(s);
2281 }
2282
2283 void RGWGetCORS_ObjStore_S3::send_response()
2284 {
2285   if (op_ret) {
2286     if (op_ret == -ENOENT)
2287       set_req_state_err(s, ERR_NO_SUCH_CORS_CONFIGURATION);
2288     else
2289       set_req_state_err(s, op_ret);
2290   }
2291   dump_errno(s);
2292   end_header(s, NULL, "application/xml");
2293   dump_start(s);
2294   if (! op_ret) {
2295     string cors;
2296     RGWCORSConfiguration_S3 *s3cors =
2297       static_cast<RGWCORSConfiguration_S3 *>(&bucket_cors);
2298     stringstream ss;
2299
2300     s3cors->to_xml(ss);
2301     cors = ss.str();
2302     dump_body(s, cors);
2303   }
2304 }
2305
2306 int RGWPutCORS_ObjStore_S3::get_params()
2307 {
2308   int r;
2309   char *data = nullptr;
2310   int len = 0;
2311   RGWCORSXMLParser_S3 parser(s->cct);
2312   RGWCORSConfiguration_S3 *cors_config;
2313
2314   const auto max_size = s->cct->_conf->rgw_max_put_param_size;
2315   r = rgw_rest_read_all_input(s, &data, &len, max_size, false);
2316   if (r < 0) {
2317     return r;
2318   }
2319
2320   auto data_deleter = std::unique_ptr<char, decltype(free)*>{data, free};
2321
2322   r = do_aws4_auth_completion();
2323   if (r < 0) {
2324     return r;
2325   }
2326
2327   if (!parser.init()) {
2328     return -EINVAL;
2329   }
2330
2331   if (!data || !parser.parse(data, len, 1)) {
2332     return -EINVAL;
2333   }
2334   cors_config =
2335     static_cast<RGWCORSConfiguration_S3 *>(parser.find_first(
2336                                              "CORSConfiguration"));
2337   if (!cors_config) {
2338     return -EINVAL;
2339   }
2340
2341   // forward bucket cors requests to meta master zone
2342   if (!store->is_meta_master()) {
2343     /* only need to keep this data around if we're not meta master */
2344     in_data.append(data, len);
2345   }
2346
2347   if (s->cct->_conf->subsys.should_gather(ceph_subsys_rgw, 15)) {
2348     ldout(s->cct, 15) << "CORSConfiguration";
2349     cors_config->to_xml(*_dout);
2350     *_dout << dendl;
2351   }
2352
2353   cors_config->encode(cors_bl);
2354
2355   return 0;
2356 }
2357
2358 void RGWPutCORS_ObjStore_S3::send_response()
2359 {
2360   if (op_ret)
2361     set_req_state_err(s, op_ret);
2362   dump_errno(s);
2363   end_header(s, NULL, "application/xml");
2364   dump_start(s);
2365 }
2366
2367 void RGWDeleteCORS_ObjStore_S3::send_response()
2368 {
2369   int r = op_ret;
2370   if (!r || r == -ENOENT)
2371     r = STATUS_NO_CONTENT;
2372
2373   set_req_state_err(s, r);
2374   dump_errno(s);
2375   end_header(s, NULL);
2376 }
2377
2378 void RGWOptionsCORS_ObjStore_S3::send_response()
2379 {
2380   string hdrs, exp_hdrs;
2381   uint32_t max_age = CORS_MAX_AGE_INVALID;
2382   /*EACCES means, there is no CORS registered yet for the bucket
2383    *ENOENT means, there is no match of the Origin in the list of CORSRule
2384    */
2385   if (op_ret == -ENOENT)
2386     op_ret = -EACCES;
2387   if (op_ret < 0) {
2388     set_req_state_err(s, op_ret);
2389     dump_errno(s);
2390     end_header(s, NULL);
2391     return;
2392   }
2393   get_response_params(hdrs, exp_hdrs, &max_age);
2394
2395   dump_errno(s);
2396   dump_access_control(s, origin, req_meth, hdrs.c_str(), exp_hdrs.c_str(),
2397                       max_age);
2398   end_header(s, NULL);
2399 }
2400
2401 void RGWGetRequestPayment_ObjStore_S3::send_response()
2402 {
2403   dump_errno(s);
2404   end_header(s, this, "application/xml");
2405   dump_start(s);
2406
2407   s->formatter->open_object_section_in_ns("RequestPaymentConfiguration", XMLNS_AWS_S3);
2408   const char *payer = requester_pays ? "Requester" :  "BucketOwner";
2409   s->formatter->dump_string("Payer", payer);
2410   s->formatter->close_section();
2411   rgw_flush_formatter_and_reset(s, s->formatter);
2412 }
2413
2414 class RGWSetRequestPaymentParser : public RGWXMLParser
2415 {
2416   XMLObj *alloc_obj(const char *el) override {
2417     return new XMLObj;
2418   }
2419
2420 public:
2421   RGWSetRequestPaymentParser() {}
2422   ~RGWSetRequestPaymentParser() override {}
2423
2424   int get_request_payment_payer(bool *requester_pays) {
2425     XMLObj *config = find_first("RequestPaymentConfiguration");
2426     if (!config)
2427       return -EINVAL;
2428
2429     *requester_pays = false;
2430
2431     XMLObj *field = config->find_first("Payer");
2432     if (!field)
2433       return 0;
2434
2435     string& s = field->get_data();
2436
2437     if (stringcasecmp(s, "Requester") == 0) {
2438       *requester_pays = true;
2439     } else if (stringcasecmp(s, "BucketOwner") != 0) {
2440       return -EINVAL;
2441     }
2442
2443     return 0;
2444   }
2445 };
2446
2447 int RGWSetRequestPayment_ObjStore_S3::get_params()
2448 {
2449   char *data;
2450   int len = 0;
2451   const auto max_size = s->cct->_conf->rgw_max_put_param_size;
2452   int r = rgw_rest_read_all_input(s, &data, &len, max_size, false);
2453
2454   if (r < 0) {
2455     return r;
2456   }
2457
2458   RGWSetRequestPaymentParser parser;
2459
2460   if (!parser.init()) {
2461     ldout(s->cct, 0) << "ERROR: failed to initialize parser" << dendl;
2462     r = -EIO;
2463     goto done;
2464   }
2465
2466   if (!parser.parse(data, len, 1)) {
2467     ldout(s->cct, 10) << "failed to parse data: " << data << dendl;
2468     r = -EINVAL;
2469     goto done;
2470   }
2471
2472   r = parser.get_request_payment_payer(&requester_pays);
2473
2474 done:
2475   free(data);
2476
2477   return r;
2478 }
2479
2480 void RGWSetRequestPayment_ObjStore_S3::send_response()
2481 {
2482   if (op_ret)
2483     set_req_state_err(s, op_ret);
2484   dump_errno(s);
2485   end_header(s);
2486 }
2487
2488 int RGWInitMultipart_ObjStore_S3::get_params()
2489 {
2490   RGWAccessControlPolicy_S3 s3policy(s->cct);
2491   op_ret = create_s3_policy(s, store, s3policy, s->owner);
2492   if (op_ret < 0)
2493     return op_ret;
2494
2495   policy = s3policy;
2496
2497   return 0;
2498 }
2499
2500 void RGWInitMultipart_ObjStore_S3::send_response()
2501 {
2502   if (op_ret)
2503     set_req_state_err(s, op_ret);
2504   dump_errno(s);
2505   for (auto &it : crypt_http_responses)
2506      dump_header(s, it.first, it.second);
2507   end_header(s, this, "application/xml");
2508   if (op_ret == 0) {
2509     dump_start(s);
2510     s->formatter->open_object_section_in_ns("InitiateMultipartUploadResult", XMLNS_AWS_S3);
2511     if (!s->bucket_tenant.empty())
2512       s->formatter->dump_string("Tenant", s->bucket_tenant);
2513     s->formatter->dump_string("Bucket", s->bucket_name);
2514     s->formatter->dump_string("Key", s->object.name);
2515     s->formatter->dump_string("UploadId", upload_id);
2516     s->formatter->close_section();
2517     rgw_flush_formatter_and_reset(s, s->formatter);
2518   }
2519 }
2520
2521 int RGWInitMultipart_ObjStore_S3::prepare_encryption(map<string, bufferlist>& attrs)
2522 {
2523   int res = 0;
2524   res = rgw_s3_prepare_encrypt(s, attrs, nullptr, nullptr, crypt_http_responses);
2525   return res;
2526 }
2527
2528 int RGWCompleteMultipart_ObjStore_S3::get_params()
2529 {
2530   int ret = RGWCompleteMultipart_ObjStore::get_params();
2531   if (ret < 0) {
2532     return ret;
2533   }
2534
2535   return do_aws4_auth_completion();
2536 }
2537
2538 void RGWCompleteMultipart_ObjStore_S3::send_response()
2539 {
2540   if (op_ret)
2541     set_req_state_err(s, op_ret);
2542   dump_errno(s);
2543   end_header(s, this, "application/xml");
2544   if (op_ret == 0) {
2545     dump_start(s);
2546     s->formatter->open_object_section_in_ns("CompleteMultipartUploadResult", XMLNS_AWS_S3);
2547     std::string base_uri = compute_domain_uri(s);
2548     if (!s->bucket_tenant.empty()) {
2549       s->formatter->dump_format("Location", "%s/%s:%s/%s",
2550           base_uri.c_str(),
2551           s->bucket_tenant.c_str(),
2552           s->bucket_name.c_str(),
2553           s->object.name.c_str()
2554           );
2555       s->formatter->dump_string("Tenant", s->bucket_tenant);
2556     } else {
2557       s->formatter->dump_format("Location", "%s/%s/%s",
2558           base_uri.c_str(),
2559           s->bucket_name.c_str(),
2560           s->object.name.c_str()
2561           );
2562     }
2563     s->formatter->dump_string("Bucket", s->bucket_name);
2564     s->formatter->dump_string("Key", s->object.name);
2565     s->formatter->dump_string("ETag", etag);
2566     s->formatter->close_section();
2567     rgw_flush_formatter_and_reset(s, s->formatter);
2568   }
2569 }
2570
2571 void RGWAbortMultipart_ObjStore_S3::send_response()
2572 {
2573   int r = op_ret;
2574   if (!r)
2575     r = STATUS_NO_CONTENT;
2576
2577   set_req_state_err(s, r);
2578   dump_errno(s);
2579   end_header(s, this);
2580 }
2581
2582 void RGWListMultipart_ObjStore_S3::send_response()
2583 {
2584   if (op_ret)
2585     set_req_state_err(s, op_ret);
2586   dump_errno(s);
2587   end_header(s, this, "application/xml");
2588
2589   if (op_ret == 0) {
2590     dump_start(s);
2591     s->formatter->open_object_section_in_ns("ListPartsResult", XMLNS_AWS_S3);
2592     map<uint32_t, RGWUploadPartInfo>::iterator iter;
2593     map<uint32_t, RGWUploadPartInfo>::reverse_iterator test_iter;
2594     int cur_max = 0;
2595
2596     iter = parts.begin();
2597     test_iter = parts.rbegin();
2598     if (test_iter != parts.rend()) {
2599       cur_max = test_iter->first;
2600     }
2601     if (!s->bucket_tenant.empty())
2602       s->formatter->dump_string("Tenant", s->bucket_tenant);
2603     s->formatter->dump_string("Bucket", s->bucket_name);
2604     s->formatter->dump_string("Key", s->object.name);
2605     s->formatter->dump_string("UploadId", upload_id);
2606     s->formatter->dump_string("StorageClass", "STANDARD");
2607     s->formatter->dump_int("PartNumberMarker", marker);
2608     s->formatter->dump_int("NextPartNumberMarker", cur_max);
2609     s->formatter->dump_int("MaxParts", max_parts);
2610     s->formatter->dump_string("IsTruncated", (truncated ? "true" : "false"));
2611
2612     ACLOwner& owner = policy.get_owner();
2613     dump_owner(s, owner.get_id(), owner.get_display_name());
2614
2615     for (; iter != parts.end(); ++iter) {
2616       RGWUploadPartInfo& info = iter->second;
2617
2618       s->formatter->open_object_section("Part");
2619
2620       dump_time(s, "LastModified", &info.modified);
2621
2622       s->formatter->dump_unsigned("PartNumber", info.num);
2623       s->formatter->dump_format("ETag", "\"%s\"", info.etag.c_str());
2624       s->formatter->dump_unsigned("Size", info.accounted_size);
2625       s->formatter->close_section();
2626     }
2627     s->formatter->close_section();
2628     rgw_flush_formatter_and_reset(s, s->formatter);
2629   }
2630 }
2631
2632 void RGWListBucketMultiparts_ObjStore_S3::send_response()
2633 {
2634   if (op_ret < 0)
2635     set_req_state_err(s, op_ret);
2636   dump_errno(s);
2637
2638   end_header(s, this, "application/xml");
2639   dump_start(s);
2640   if (op_ret < 0)
2641     return;
2642
2643   s->formatter->open_object_section_in_ns("ListMultipartUploadsResult", XMLNS_AWS_S3);
2644   if (!s->bucket_tenant.empty())
2645     s->formatter->dump_string("Tenant", s->bucket_tenant);
2646   s->formatter->dump_string("Bucket", s->bucket_name);
2647   if (!prefix.empty())
2648     s->formatter->dump_string("ListMultipartUploadsResult.Prefix", prefix);
2649   string& key_marker = marker.get_key();
2650   if (!key_marker.empty())
2651     s->formatter->dump_string("KeyMarker", key_marker);
2652   string& upload_id_marker = marker.get_upload_id();
2653   if (!upload_id_marker.empty())
2654     s->formatter->dump_string("UploadIdMarker", upload_id_marker);
2655   string next_key = next_marker.mp.get_key();
2656   if (!next_key.empty())
2657     s->formatter->dump_string("NextKeyMarker", next_key);
2658   string next_upload_id = next_marker.mp.get_upload_id();
2659   if (!next_upload_id.empty())
2660     s->formatter->dump_string("NextUploadIdMarker", next_upload_id);
2661   s->formatter->dump_int("MaxUploads", max_uploads);
2662   if (!delimiter.empty())
2663     s->formatter->dump_string("Delimiter", delimiter);
2664   s->formatter->dump_string("IsTruncated", (is_truncated ? "true" : "false"));
2665
2666   if (op_ret >= 0) {
2667     vector<RGWMultipartUploadEntry>::iterator iter;
2668     for (iter = uploads.begin(); iter != uploads.end(); ++iter) {
2669       RGWMPObj& mp = iter->mp;
2670       s->formatter->open_array_section("Upload");
2671       s->formatter->dump_string("Key", mp.get_key());
2672       s->formatter->dump_string("UploadId", mp.get_upload_id());
2673       dump_owner(s, s->user->user_id, s->user->display_name, "Initiator");
2674       dump_owner(s, s->user->user_id, s->user->display_name);
2675       s->formatter->dump_string("StorageClass", "STANDARD");
2676       dump_time(s, "Initiated", &iter->obj.meta.mtime);
2677       s->formatter->close_section();
2678     }
2679     if (!common_prefixes.empty()) {
2680       s->formatter->open_array_section("CommonPrefixes");
2681       map<string, bool>::iterator pref_iter;
2682       for (pref_iter = common_prefixes.begin();
2683            pref_iter != common_prefixes.end(); ++pref_iter) {
2684         s->formatter->dump_string("CommonPrefixes.Prefix", pref_iter->first);
2685       }
2686       s->formatter->close_section();
2687     }
2688   }
2689   s->formatter->close_section();
2690   rgw_flush_formatter_and_reset(s, s->formatter);
2691 }
2692
2693 int RGWDeleteMultiObj_ObjStore_S3::get_params()
2694 {
2695   int ret = RGWDeleteMultiObj_ObjStore::get_params();
2696   if (ret < 0) {
2697     return ret;
2698   }
2699
2700   return do_aws4_auth_completion();
2701 }
2702
2703 void RGWDeleteMultiObj_ObjStore_S3::send_status()
2704 {
2705   if (! status_dumped) {
2706     if (op_ret < 0)
2707       set_req_state_err(s, op_ret);
2708     dump_errno(s);
2709     status_dumped = true;
2710   }
2711 }
2712
2713 void RGWDeleteMultiObj_ObjStore_S3::begin_response()
2714 {
2715
2716   if (!status_dumped) {
2717     send_status();
2718   }
2719
2720   dump_start(s);
2721   end_header(s, this, "application/xml");
2722   s->formatter->open_object_section_in_ns("DeleteResult", XMLNS_AWS_S3);
2723
2724   rgw_flush_formatter(s, s->formatter);
2725 }
2726
2727 void RGWDeleteMultiObj_ObjStore_S3::send_partial_response(rgw_obj_key& key,
2728                                                           bool delete_marker,
2729                                                           const string& marker_version_id, int ret)
2730 {
2731   if (!key.empty()) {
2732     if (op_ret == 0 && !quiet) {
2733       s->formatter->open_object_section("Deleted");
2734       s->formatter->dump_string("Key", key.name);
2735       if (!key.instance.empty()) {
2736         s->formatter->dump_string("VersionId", key.instance);
2737       }
2738       if (delete_marker) {
2739         s->formatter->dump_bool("DeleteMarker", true);
2740         s->formatter->dump_string("DeleteMarkerVersionId", marker_version_id);
2741       }
2742       s->formatter->close_section();
2743     } else if (op_ret < 0) {
2744       struct rgw_http_error r;
2745       int err_no;
2746
2747       s->formatter->open_object_section("Error");
2748
2749       err_no = -op_ret;
2750       rgw_get_errno_s3(&r, err_no);
2751
2752       s->formatter->dump_string("Key", key.name);
2753       s->formatter->dump_string("VersionId", key.instance);
2754       s->formatter->dump_int("Code", r.http_ret);
2755       s->formatter->dump_string("Message", r.s3_code);
2756       s->formatter->close_section();
2757     }
2758
2759     rgw_flush_formatter(s, s->formatter);
2760   }
2761 }
2762
2763 void RGWDeleteMultiObj_ObjStore_S3::end_response()
2764 {
2765
2766   s->formatter->close_section();
2767   rgw_flush_formatter_and_reset(s, s->formatter);
2768 }
2769
2770 void RGWGetObjLayout_ObjStore_S3::send_response()
2771 {
2772   if (op_ret)
2773     set_req_state_err(s, op_ret);
2774   dump_errno(s);
2775   end_header(s, this, "application/json");
2776
2777   JSONFormatter f;
2778
2779   if (op_ret < 0) {
2780     return;
2781   }
2782
2783   f.open_object_section("result");
2784   ::encode_json("head", head_obj, &f);
2785   ::encode_json("manifest", *manifest, &f);
2786   f.open_array_section("data_location");
2787   for (auto miter = manifest->obj_begin(); miter != manifest->obj_end(); ++miter) {
2788     f.open_object_section("obj");
2789     rgw_raw_obj raw_loc = miter.get_location().get_raw_obj(store);
2790     ::encode_json("ofs", miter.get_ofs(), &f);
2791     ::encode_json("loc", raw_loc, &f);
2792     ::encode_json("loc_ofs", miter.location_ofs(), &f);
2793     ::encode_json("loc_size", miter.get_stripe_size(), &f);
2794     f.close_section();
2795     rgw_flush_formatter(s, &f);
2796   }
2797   f.close_section();
2798   f.close_section();
2799   rgw_flush_formatter(s, &f);
2800 }
2801
2802 int RGWConfigBucketMetaSearch_ObjStore_S3::get_params()
2803 {
2804   auto iter = s->info.x_meta_map.find("x-amz-meta-search");
2805   if (iter == s->info.x_meta_map.end()) {
2806     s->err.message = "X-Rgw-Meta-Search header not provided";
2807     ldout(s->cct, 5) << s->err.message << dendl;
2808     return -EINVAL;
2809   }
2810
2811   list<string> expressions;
2812   get_str_list(iter->second, ",", expressions);
2813
2814   for (auto& expression : expressions) {
2815     vector<string> args;
2816     get_str_vec(expression, ";", args);
2817
2818     if (args.empty()) {
2819       s->err.message = "invalid empty expression";
2820       ldout(s->cct, 5) << s->err.message << dendl;
2821       return -EINVAL;
2822     }
2823     if (args.size() > 2) {
2824       s->err.message = string("invalid expression: ") + expression;
2825       ldout(s->cct, 5) << s->err.message << dendl;
2826       return -EINVAL;
2827     }
2828
2829     string key = boost::algorithm::to_lower_copy(rgw_trim_whitespace(args[0]));
2830     string val;
2831     if (args.size() > 1) {
2832       val = boost::algorithm::to_lower_copy(rgw_trim_whitespace(args[1]));
2833     }
2834
2835     if (!boost::algorithm::starts_with(key, RGW_AMZ_META_PREFIX)) {
2836       s->err.message = string("invalid expression, key must start with '" RGW_AMZ_META_PREFIX "' : ") + expression;
2837       ldout(s->cct, 5) << s->err.message << dendl;
2838       return -EINVAL;
2839     }
2840
2841     key = key.substr(sizeof(RGW_AMZ_META_PREFIX) - 1);
2842
2843     ESEntityTypeMap::EntityType entity_type;
2844
2845     if (val.empty() || val == "str" || val == "string") {
2846       entity_type = ESEntityTypeMap::ES_ENTITY_STR;
2847     } else if (val == "int" || val == "integer") {
2848       entity_type = ESEntityTypeMap::ES_ENTITY_INT;
2849     } else if (val == "date" || val == "datetime") {
2850       entity_type = ESEntityTypeMap::ES_ENTITY_DATE;
2851     } else {
2852       s->err.message = string("invalid entity type: ") + val;
2853       ldout(s->cct, 5) << s->err.message << dendl;
2854       return -EINVAL;
2855     }
2856
2857     mdsearch_config[key] = entity_type;
2858   }
2859
2860   return 0;
2861 }
2862
2863 void RGWConfigBucketMetaSearch_ObjStore_S3::send_response()
2864 {
2865   if (op_ret)
2866     set_req_state_err(s, op_ret);
2867   dump_errno(s);
2868   end_header(s, this);
2869 }
2870
2871 void RGWGetBucketMetaSearch_ObjStore_S3::send_response()
2872 {
2873   if (op_ret)
2874     set_req_state_err(s, op_ret);
2875   dump_errno(s);
2876   end_header(s, NULL, "application/xml");
2877
2878   Formatter *f = s->formatter;
2879   f->open_array_section("GetBucketMetaSearchResult");
2880   for (auto& e : s->bucket_info.mdsearch_config) {
2881     f->open_object_section("Entry");
2882     string k = string("x-amz-meta-") + e.first;
2883     f->dump_string("Key", k.c_str());
2884     const char *type;
2885     switch (e.second) {
2886       case ESEntityTypeMap::ES_ENTITY_INT:
2887         type = "int";
2888         break;
2889       case ESEntityTypeMap::ES_ENTITY_DATE:
2890         type = "date";
2891         break;
2892       default:
2893         type = "str";
2894     }
2895     f->dump_string("Type", type);
2896     f->close_section();
2897   }
2898   f->close_section();
2899   rgw_flush_formatter(s, f);
2900 }
2901
2902 void RGWDelBucketMetaSearch_ObjStore_S3::send_response()
2903 {
2904   if (op_ret)
2905     set_req_state_err(s, op_ret);
2906   dump_errno(s);
2907   end_header(s, this);
2908 }
2909
2910
2911 RGWOp *RGWHandler_REST_Service_S3::op_get()
2912 {
2913   if (is_usage_op()) {
2914     return new RGWGetUsage_ObjStore_S3;
2915   } else {
2916     return new RGWListBuckets_ObjStore_S3;
2917   }
2918 }
2919
2920 RGWOp *RGWHandler_REST_Service_S3::op_head()
2921 {
2922   return new RGWListBuckets_ObjStore_S3;
2923 }
2924
2925 RGWOp *RGWHandler_REST_Service_S3::op_post()
2926 {
2927   if (s->info.args.exists("Action")) {
2928     string action = s->info.args.get("Action");
2929     if (action.compare("CreateRole") == 0)
2930       return new RGWCreateRole;
2931     if (action.compare("DeleteRole") == 0)
2932       return new RGWDeleteRole;
2933     if (action.compare("GetRole") == 0)
2934       return new RGWGetRole;
2935     if (action.compare("UpdateAssumeRolePolicy") == 0)
2936       return new RGWModifyRole;
2937     if (action.compare("ListRoles") == 0)
2938       return new RGWListRoles;
2939     if (action.compare("PutRolePolicy") == 0)
2940       return new RGWPutRolePolicy;
2941     if (action.compare("GetRolePolicy") == 0)
2942       return new RGWGetRolePolicy;
2943     if (action.compare("ListRolePolicies") == 0)
2944       return new RGWListRolePolicies;
2945     if (action.compare("DeleteRolePolicy") == 0)
2946       return new RGWDeleteRolePolicy;
2947   }
2948   return NULL;
2949 }
2950
2951 RGWOp *RGWHandler_REST_Bucket_S3::get_obj_op(bool get_data)
2952 {
2953   // Non-website mode
2954   if (get_data) {
2955     return new RGWListBucket_ObjStore_S3;
2956   } else {
2957     return new RGWStatBucket_ObjStore_S3;
2958   }
2959 }
2960
2961 RGWOp *RGWHandler_REST_Bucket_S3::op_get()
2962 {
2963   if (s->info.args.sub_resource_exists("logging"))
2964     return new RGWGetBucketLogging_ObjStore_S3;
2965
2966   if (s->info.args.sub_resource_exists("location"))
2967     return new RGWGetBucketLocation_ObjStore_S3;
2968
2969   if (s->info.args.sub_resource_exists("versioning"))
2970     return new RGWGetBucketVersioning_ObjStore_S3;
2971
2972   if (s->info.args.sub_resource_exists("website")) {
2973     if (!s->cct->_conf->rgw_enable_static_website) {
2974       return NULL;
2975     }
2976     return new RGWGetBucketWebsite_ObjStore_S3;
2977   }
2978
2979   if (s->info.args.exists("mdsearch")) {
2980     return new RGWGetBucketMetaSearch_ObjStore_S3;
2981   }
2982
2983   if (is_acl_op()) {
2984     return new RGWGetACLs_ObjStore_S3;
2985   } else if (is_cors_op()) {
2986     return new RGWGetCORS_ObjStore_S3;
2987   } else if (is_request_payment_op()) {
2988     return new RGWGetRequestPayment_ObjStore_S3;
2989   } else if (s->info.args.exists("uploads")) {
2990     return new RGWListBucketMultiparts_ObjStore_S3;
2991   } else if(is_lc_op()) {
2992     return new RGWGetLC_ObjStore_S3;
2993   } else if(is_policy_op()) {
2994     return new RGWGetBucketPolicy;
2995   }
2996   return get_obj_op(true);
2997 }
2998
2999 RGWOp *RGWHandler_REST_Bucket_S3::op_head()
3000 {
3001   if (is_acl_op()) {
3002     return new RGWGetACLs_ObjStore_S3;
3003   } else if (s->info.args.exists("uploads")) {
3004     return new RGWListBucketMultiparts_ObjStore_S3;
3005   }
3006   return get_obj_op(false);
3007 }
3008
3009 RGWOp *RGWHandler_REST_Bucket_S3::op_put()
3010 {
3011   if (s->info.args.sub_resource_exists("logging"))
3012     return NULL;
3013   if (s->info.args.sub_resource_exists("versioning"))
3014     return new RGWSetBucketVersioning_ObjStore_S3;
3015   if (s->info.args.sub_resource_exists("website")) {
3016     if (!s->cct->_conf->rgw_enable_static_website) {
3017       return NULL;
3018     }
3019     return new RGWSetBucketWebsite_ObjStore_S3;
3020   }
3021   if (is_acl_op()) {
3022     return new RGWPutACLs_ObjStore_S3;
3023   } else if (is_cors_op()) {
3024     return new RGWPutCORS_ObjStore_S3;
3025   } else if (is_request_payment_op()) {
3026     return new RGWSetRequestPayment_ObjStore_S3;
3027   } else if(is_lc_op()) {
3028     return new RGWPutLC_ObjStore_S3;
3029   } else if(is_policy_op()) {
3030     return new RGWPutBucketPolicy;
3031   }
3032   return new RGWCreateBucket_ObjStore_S3;
3033 }
3034
3035 RGWOp *RGWHandler_REST_Bucket_S3::op_delete()
3036 {
3037   if (is_cors_op()) {
3038     return new RGWDeleteCORS_ObjStore_S3;
3039   } else if(is_lc_op()) {
3040     return new RGWDeleteLC_ObjStore_S3;
3041   } else if(is_policy_op()) {
3042     return new RGWDeleteBucketPolicy;
3043   }
3044
3045   if (s->info.args.sub_resource_exists("website")) {
3046     if (!s->cct->_conf->rgw_enable_static_website) {
3047       return NULL;
3048     }
3049     return new RGWDeleteBucketWebsite_ObjStore_S3;
3050   }
3051
3052   if (s->info.args.exists("mdsearch")) {
3053     return new RGWDelBucketMetaSearch_ObjStore_S3;
3054   }
3055
3056   return new RGWDeleteBucket_ObjStore_S3;
3057 }
3058
3059 RGWOp *RGWHandler_REST_Bucket_S3::op_post()
3060 {
3061   if (s->info.args.exists("delete")) {
3062     return new RGWDeleteMultiObj_ObjStore_S3;
3063   }
3064
3065   if (s->info.args.exists("mdsearch")) {
3066     return new RGWConfigBucketMetaSearch_ObjStore_S3;
3067   }
3068
3069   return new RGWPostObj_ObjStore_S3;
3070 }
3071
3072 RGWOp *RGWHandler_REST_Bucket_S3::op_options()
3073 {
3074   return new RGWOptionsCORS_ObjStore_S3;
3075 }
3076
3077 RGWOp *RGWHandler_REST_Obj_S3::get_obj_op(bool get_data)
3078 {
3079   if (is_acl_op()) {
3080     return new RGWGetACLs_ObjStore_S3;
3081   }
3082   RGWGetObj_ObjStore_S3 *get_obj_op = new RGWGetObj_ObjStore_S3;
3083   get_obj_op->set_get_data(get_data);
3084   return get_obj_op;
3085 }
3086
3087 RGWOp *RGWHandler_REST_Obj_S3::op_get()
3088 {
3089   if (is_acl_op()) {
3090     return new RGWGetACLs_ObjStore_S3;
3091   } else if (s->info.args.exists("uploadId")) {
3092     return new RGWListMultipart_ObjStore_S3;
3093   } else if (s->info.args.exists("layout")) {
3094     return new RGWGetObjLayout_ObjStore_S3;
3095   } else if (is_tagging_op()) {
3096     return new RGWGetObjTags_ObjStore_S3;
3097   }
3098   return get_obj_op(true);
3099 }
3100
3101 RGWOp *RGWHandler_REST_Obj_S3::op_head()
3102 {
3103   if (is_acl_op()) {
3104     return new RGWGetACLs_ObjStore_S3;
3105   } else if (s->info.args.exists("uploadId")) {
3106     return new RGWListMultipart_ObjStore_S3;
3107   }
3108   return get_obj_op(false);
3109 }
3110
3111 RGWOp *RGWHandler_REST_Obj_S3::op_put()
3112 {
3113   if (is_acl_op()) {
3114     return new RGWPutACLs_ObjStore_S3;
3115   } else if (is_tagging_op()) {
3116     return new RGWPutObjTags_ObjStore_S3;
3117   }
3118
3119   if (s->init_state.src_bucket.empty())
3120     return new RGWPutObj_ObjStore_S3;
3121   else
3122     return new RGWCopyObj_ObjStore_S3;
3123 }
3124
3125 RGWOp *RGWHandler_REST_Obj_S3::op_delete()
3126 {
3127   if (is_tagging_op()) {
3128     return new RGWDeleteObjTags_ObjStore_S3;
3129   }
3130   string upload_id = s->info.args.get("uploadId");
3131
3132   if (upload_id.empty())
3133     return new RGWDeleteObj_ObjStore_S3;
3134   else
3135     return new RGWAbortMultipart_ObjStore_S3;
3136 }
3137
3138 RGWOp *RGWHandler_REST_Obj_S3::op_post()
3139 {
3140   if (s->info.args.exists("uploadId"))
3141     return new RGWCompleteMultipart_ObjStore_S3;
3142
3143   if (s->info.args.exists("uploads"))
3144     return new RGWInitMultipart_ObjStore_S3;
3145
3146   return new RGWPostObj_ObjStore_S3;
3147 }
3148
3149 RGWOp *RGWHandler_REST_Obj_S3::op_options()
3150 {
3151   return new RGWOptionsCORS_ObjStore_S3;
3152 }
3153
3154 int RGWHandler_REST_S3::init_from_header(struct req_state* s,
3155                                         int default_formatter,
3156                                         bool configurable_format)
3157 {
3158   string req;
3159   string first;
3160
3161   const char *req_name = s->relative_uri.c_str();
3162   const char *p;
3163
3164   if (*req_name == '?') {
3165     p = req_name;
3166   } else {
3167     p = s->info.request_params.c_str();
3168   }
3169
3170   s->info.args.set(p);
3171   s->info.args.parse();
3172
3173   /* must be called after the args parsing */
3174   int ret = allocate_formatter(s, default_formatter, configurable_format);
3175   if (ret < 0)
3176     return ret;
3177
3178   if (*req_name != '/')
3179     return 0;
3180
3181   req_name++;
3182
3183   if (!*req_name)
3184     return 0;
3185
3186   req = req_name;
3187   int pos = req.find('/');
3188   if (pos >= 0) {
3189     first = req.substr(0, pos);
3190   } else {
3191     first = req;
3192   }
3193
3194   /*
3195    * XXX The intent of the check for empty is apparently to let the bucket
3196    * name from DNS to be set ahead. However, we currently take the DNS
3197    * bucket and re-insert it into URL in rgw_rest.cc:RGWREST::preprocess().
3198    * So, this check is meaningless.
3199    *
3200    * Rather than dropping this, the code needs to be changed into putting
3201    * the bucket (and its tenant) from DNS and Host: header (HTTP_HOST)
3202    * into req_status.bucket_name directly.
3203    */
3204   if (s->init_state.url_bucket.empty()) {
3205     // Save bucket to tide us over until token is parsed.
3206     s->init_state.url_bucket = first;
3207     if (pos >= 0) {
3208       string encoded_obj_str = req.substr(pos+1);
3209       s->object = rgw_obj_key(encoded_obj_str, s->info.args.get("versionId"));
3210     }
3211   } else {
3212     s->object = rgw_obj_key(req_name, s->info.args.get("versionId"));
3213   }
3214   return 0;
3215 }
3216
3217 int RGWHandler_REST_S3::postauth_init()
3218 {
3219   struct req_init_state *t = &s->init_state;
3220   bool relaxed_names = s->cct->_conf->rgw_relaxed_s3_bucket_names;
3221
3222   rgw_parse_url_bucket(t->url_bucket, s->user->user_id.tenant,
3223                       s->bucket_tenant, s->bucket_name);
3224
3225   dout(10) << "s->object=" << (!s->object.empty() ? s->object : rgw_obj_key("<NULL>"))
3226            << " s->bucket=" << rgw_make_bucket_entry_name(s->bucket_tenant, s->bucket_name) << dendl;
3227
3228   int ret;
3229   ret = rgw_validate_tenant_name(s->bucket_tenant);
3230   if (ret)
3231     return ret;
3232   if (!s->bucket_name.empty()) {
3233     ret = valid_s3_bucket_name(s->bucket_name, relaxed_names);
3234     if (ret)
3235       return ret;
3236     ret = validate_object_name(s->object.name);
3237     if (ret)
3238       return ret;
3239   }
3240
3241   if (!t->src_bucket.empty()) {
3242     rgw_parse_url_bucket(t->src_bucket, s->user->user_id.tenant,
3243                         s->src_tenant_name, s->src_bucket_name);
3244     ret = rgw_validate_tenant_name(s->src_tenant_name);
3245     if (ret)
3246       return ret;
3247     ret = valid_s3_bucket_name(s->src_bucket_name, relaxed_names);
3248     if (ret)
3249       return ret;
3250   }
3251   return 0;
3252 }
3253
3254 int RGWHandler_REST_S3::init(RGWRados *store, struct req_state *s,
3255                              rgw::io::BasicClient *cio)
3256 {
3257   int ret;
3258
3259   s->dialect = "s3";
3260
3261   ret = rgw_validate_tenant_name(s->bucket_tenant);
3262   if (ret)
3263     return ret;
3264   bool relaxed_names = s->cct->_conf->rgw_relaxed_s3_bucket_names;
3265   if (!s->bucket_name.empty()) {
3266     ret = valid_s3_bucket_name(s->bucket_name, relaxed_names);
3267     if (ret)
3268       return ret;
3269     ret = validate_object_name(s->object.name);
3270     if (ret)
3271       return ret;
3272   }
3273
3274   const char *cacl = s->info.env->get("HTTP_X_AMZ_ACL");
3275   if (cacl)
3276     s->canned_acl = cacl;
3277
3278   s->has_acl_header = s->info.env->exists_prefix("HTTP_X_AMZ_GRANT");
3279
3280   const char *copy_source = s->info.env->get("HTTP_X_AMZ_COPY_SOURCE");
3281   if (copy_source &&
3282       (! s->info.env->get("HTTP_X_AMZ_COPY_SOURCE_RANGE")) &&
3283       (! s->info.args.exists("uploadId"))) {
3284
3285     ret = RGWCopyObj::parse_copy_location(url_decode(copy_source),
3286                                           s->init_state.src_bucket,
3287                                           s->src_object);
3288     if (!ret) {
3289       ldout(s->cct, 0) << "failed to parse copy location" << dendl;
3290       return -EINVAL; // XXX why not -ERR_INVALID_BUCKET_NAME or -ERR_BAD_URL?
3291     }
3292   }
3293
3294   return RGWHandler_REST::init(store, s, cio);
3295 }
3296
3297 enum class AwsVersion {
3298   UNKOWN,
3299   V2,
3300   V4
3301 };
3302
3303 enum class AwsRoute {
3304   UNKOWN,
3305   QUERY_STRING,
3306   HEADERS
3307 };
3308
3309 static inline std::pair<AwsVersion, AwsRoute>
3310 discover_aws_flavour(const req_info& info)
3311 {
3312   using rgw::auth::s3::AWS4_HMAC_SHA256_STR;
3313
3314   AwsVersion version = AwsVersion::UNKOWN;
3315   AwsRoute route = AwsRoute::UNKOWN;
3316
3317   const char* http_auth = info.env->get("HTTP_AUTHORIZATION");
3318   if (http_auth && http_auth[0]) {
3319     /* Authorization in Header */
3320     route = AwsRoute::HEADERS;
3321
3322     if (!strncmp(http_auth, AWS4_HMAC_SHA256_STR,
3323                  strlen(AWS4_HMAC_SHA256_STR))) {
3324       /* AWS v4 */
3325       version = AwsVersion::V4;
3326     } else if (!strncmp(http_auth, "AWS ", 4)) {
3327       /* AWS v2 */
3328       version = AwsVersion::V2;
3329     }
3330   } else {
3331     route = AwsRoute::QUERY_STRING;
3332
3333     if (info.args.get("X-Amz-Algorithm") == AWS4_HMAC_SHA256_STR) {
3334       /* AWS v4 */
3335       version = AwsVersion::V4;
3336     } else if (!info.args.get("AWSAccessKeyId").empty()) {
3337       /* AWS v2 */
3338       version = AwsVersion::V2;
3339     }
3340   }
3341
3342   return std::make_pair(version, route);
3343 }
3344
3345 static void init_anon_user(struct req_state *s)
3346 {
3347   rgw_get_anon_user(*(s->user));
3348   s->perm_mask = RGW_PERM_FULL_CONTROL;
3349 }
3350
3351 /*
3352  * verify that a signed request comes from the keyholder
3353  * by checking the signature against our locally-computed version
3354  *
3355  * it tries AWS v4 before AWS v2
3356  */
3357 int RGW_Auth_S3::authorize(RGWRados* const store,
3358                            const rgw::auth::StrategyRegistry& auth_registry,
3359                            struct req_state* const s)
3360 {
3361
3362   /* neither keystone and rados enabled; warn and exit! */
3363   if (!store->ctx()->_conf->rgw_s3_auth_use_rados &&
3364       !store->ctx()->_conf->rgw_s3_auth_use_keystone &&
3365       !store->ctx()->_conf->rgw_s3_auth_use_ldap) {
3366     dout(0) << "WARNING: no authorization backend enabled! Users will never authenticate." << dendl;
3367     return -EPERM;
3368   }
3369
3370   const auto ret = rgw::auth::Strategy::apply(auth_registry.get_s3_main(), s);
3371   if (ret == 0) {
3372     /* Populate the owner info. */
3373     s->owner.set_id(s->user->user_id);
3374     s->owner.set_name(s->user->display_name);
3375   }
3376   return ret;
3377 }
3378
3379 int RGWHandler_Auth_S3::init(RGWRados *store, struct req_state *state,
3380                              rgw::io::BasicClient *cio)
3381 {
3382   int ret = RGWHandler_REST_S3::init_from_header(state, RGW_FORMAT_JSON,
3383                                                      true);
3384   if (ret < 0)
3385     return ret;
3386
3387   return RGWHandler_REST::init(store, state, cio);
3388 }
3389
3390 RGWHandler_REST* RGWRESTMgr_S3::get_handler(struct req_state* const s,
3391                                             const rgw::auth::StrategyRegistry& auth_registry,
3392                                             const std::string& frontend_prefix)
3393 {
3394   bool is_s3website = enable_s3website && (s->prot_flags & RGW_REST_WEBSITE);
3395   int ret =
3396     RGWHandler_REST_S3::init_from_header(s,
3397                                         is_s3website ? RGW_FORMAT_HTML :
3398                                         RGW_FORMAT_XML, true);
3399   if (ret < 0)
3400     return NULL;
3401
3402   RGWHandler_REST* handler;
3403   // TODO: Make this more readable
3404   if (is_s3website) {
3405     if (s->init_state.url_bucket.empty()) {
3406       handler = new RGWHandler_REST_Service_S3Website(auth_registry);
3407     } else if (s->object.empty()) {
3408       handler = new RGWHandler_REST_Bucket_S3Website(auth_registry);
3409     } else {
3410       handler = new RGWHandler_REST_Obj_S3Website(auth_registry);
3411     }
3412   } else {
3413     if (s->init_state.url_bucket.empty()) {
3414       handler = new RGWHandler_REST_Service_S3(auth_registry);
3415     } else if (s->object.empty()) {
3416       handler = new RGWHandler_REST_Bucket_S3(auth_registry);
3417     } else {
3418       handler = new RGWHandler_REST_Obj_S3(auth_registry);
3419     }
3420   }
3421
3422   ldout(s->cct, 20) << __func__ << " handler=" << typeid(*handler).name()
3423                     << dendl;
3424   return handler;
3425 }
3426
3427 bool RGWHandler_REST_S3Website::web_dir() const {
3428   std::string subdir_name = url_decode(s->object.name);
3429
3430   if (subdir_name.empty()) {
3431     return false;
3432   } else if (subdir_name.back() == '/') {
3433     subdir_name.pop_back();
3434   }
3435
3436   rgw_obj obj(s->bucket, subdir_name);
3437
3438   RGWObjectCtx& obj_ctx = *static_cast<RGWObjectCtx *>(s->obj_ctx);
3439   obj_ctx.obj.set_atomic(obj);
3440   obj_ctx.obj.set_prefetch_data(obj);
3441
3442   RGWObjState* state = nullptr;
3443   if (store->get_obj_state(&obj_ctx, s->bucket_info, obj, &state, false) < 0) {
3444     return false;
3445   }
3446   if (! state->exists) {
3447     return false;
3448   }
3449   return state->exists;
3450 }
3451
3452 int RGWHandler_REST_S3Website::init(RGWRados *store, req_state *s,
3453                                     rgw::io::BasicClient* cio)
3454 {
3455   // save the original object name before retarget() replaces it with the
3456   // result of get_effective_key(). the error_handler() needs the original
3457   // object name for redirect handling
3458   original_object_name = s->object.name;
3459
3460   return RGWHandler_REST_S3::init(store, s, cio);
3461 }
3462
3463 int RGWHandler_REST_S3Website::retarget(RGWOp* op, RGWOp** new_op) {
3464   *new_op = op;
3465   ldout(s->cct, 10) << __func__ << "Starting retarget" << dendl;
3466
3467   if (!(s->prot_flags & RGW_REST_WEBSITE))
3468     return 0;
3469
3470   RGWObjectCtx& obj_ctx = *static_cast<RGWObjectCtx *>(s->obj_ctx);
3471   int ret = store->get_bucket_info(obj_ctx, s->bucket_tenant,
3472                                   s->bucket_name, s->bucket_info, NULL,
3473                                   &s->bucket_attrs);
3474   if (ret < 0) {
3475       // TODO-FUTURE: if the bucket does not exist, maybe expose it here?
3476       return -ERR_NO_SUCH_BUCKET;
3477   }
3478   if (!s->bucket_info.has_website) {
3479       // TODO-FUTURE: if the bucket has no WebsiteConfig, expose it here
3480       return -ERR_NO_SUCH_WEBSITE_CONFIGURATION;
3481   }
3482
3483   rgw_obj_key new_obj;
3484   s->bucket_info.website_conf.get_effective_key(s->object.name, &new_obj.name, web_dir());
3485   ldout(s->cct, 10) << "retarget get_effective_key " << s->object << " -> "
3486                     << new_obj << dendl;
3487
3488   RGWBWRoutingRule rrule;
3489   bool should_redirect =
3490     s->bucket_info.website_conf.should_redirect(new_obj.name, 0, &rrule);
3491
3492   if (should_redirect) {
3493     const string& hostname = s->info.env->get("HTTP_HOST", "");
3494     const string& protocol =
3495       (s->info.env->get("SERVER_PORT_SECURE") ? "https" : "http");
3496     int redirect_code = 0;
3497     rrule.apply_rule(protocol, hostname, s->object.name, &s->redirect,
3498                     &redirect_code);
3499     // APply a custom HTTP response code
3500     if (redirect_code > 0)
3501       s->err.http_ret = redirect_code; // Apply a custom HTTP response code
3502     ldout(s->cct, 10) << "retarget redirect code=" << redirect_code
3503                       << " proto+host:" << protocol << "://" << hostname
3504                       << " -> " << s->redirect << dendl;
3505     return -ERR_WEBSITE_REDIRECT;
3506   }
3507
3508   /*
3509    * FIXME: if s->object != new_obj, drop op and create a new op to handle
3510    * operation. Or remove this comment if it's not applicable anymore
3511    */
3512
3513   s->object = new_obj;
3514
3515   return 0;
3516 }
3517
3518 RGWOp* RGWHandler_REST_S3Website::op_get()
3519 {
3520   return get_obj_op(true);
3521 }
3522
3523 RGWOp* RGWHandler_REST_S3Website::op_head()
3524 {
3525   return get_obj_op(false);
3526 }
3527
3528 int RGWHandler_REST_S3Website::serve_errordoc(int http_ret, const string& errordoc_key) {
3529   int ret = 0;
3530   s->formatter->reset(); /* Try to throw it all away */
3531
3532   std::shared_ptr<RGWGetObj_ObjStore_S3Website> getop( static_cast<RGWGetObj_ObjStore_S3Website*>(op_get()));
3533   if (getop.get() == NULL) {
3534     return -1; // Trigger double error handler
3535   }
3536   getop->init(store, s, this);
3537   getop->range_str = NULL;
3538   getop->if_mod = NULL;
3539   getop->if_unmod = NULL;
3540   getop->if_match = NULL;
3541   getop->if_nomatch = NULL;
3542   s->object = errordoc_key;
3543
3544   ret = init_permissions(getop.get());
3545   if (ret < 0) {
3546     ldout(s->cct, 20) << "serve_errordoc failed, init_permissions ret=" << ret << dendl;
3547     return -1; // Trigger double error handler
3548   }
3549
3550   ret = read_permissions(getop.get());
3551   if (ret < 0) {
3552     ldout(s->cct, 20) << "serve_errordoc failed, read_permissions ret=" << ret << dendl;
3553     return -1; // Trigger double error handler
3554   }
3555
3556   if (http_ret) {
3557      getop->set_custom_http_response(http_ret);
3558   }
3559
3560   ret = getop->init_processing();
3561   if (ret < 0) {
3562     ldout(s->cct, 20) << "serve_errordoc failed, init_processing ret=" << ret << dendl;
3563     return -1; // Trigger double error handler
3564   }
3565
3566   ret = getop->verify_op_mask();
3567   if (ret < 0) {
3568     ldout(s->cct, 20) << "serve_errordoc failed, verify_op_mask ret=" << ret << dendl;
3569     return -1; // Trigger double error handler
3570   }
3571
3572   ret = getop->verify_permission();
3573   if (ret < 0) {
3574     ldout(s->cct, 20) << "serve_errordoc failed, verify_permission ret=" << ret << dendl;
3575     return -1; // Trigger double error handler
3576   }
3577
3578   ret = getop->verify_params();
3579   if (ret < 0) {
3580     ldout(s->cct, 20) << "serve_errordoc failed, verify_params ret=" << ret << dendl;
3581     return -1; // Trigger double error handler
3582   }
3583
3584   // No going back now
3585   getop->pre_exec();
3586   /*
3587    * FIXME Missing headers:
3588    * With a working errordoc, the s3 error fields are rendered as HTTP headers,
3589    *   x-amz-error-code: NoSuchKey
3590    *   x-amz-error-message: The specified key does not exist.
3591    *   x-amz-error-detail-Key: foo
3592    */
3593   getop->execute();
3594   getop->complete();
3595   return 0;
3596
3597 }
3598
3599 int RGWHandler_REST_S3Website::error_handler(int err_no,
3600                                             string* error_content) {
3601   int new_err_no = -1;
3602   rgw_http_errors::const_iterator r = rgw_http_s3_errors.find(err_no > 0 ? err_no : -err_no);
3603   int http_error_code = -1;
3604
3605   if (r != rgw_http_s3_errors.end()) {
3606     http_error_code = r->second.first;
3607   }
3608   ldout(s->cct, 10) << "RGWHandler_REST_S3Website::error_handler err_no=" << err_no << " http_ret=" << http_error_code << dendl;
3609
3610   RGWBWRoutingRule rrule;
3611   bool should_redirect =
3612     s->bucket_info.website_conf.should_redirect(original_object_name,
3613                                                 http_error_code, &rrule);
3614
3615   if (should_redirect) {
3616     const string& hostname = s->info.env->get("HTTP_HOST", "");
3617     const string& protocol =
3618       (s->info.env->get("SERVER_PORT_SECURE") ? "https" : "http");
3619     int redirect_code = 0;
3620     rrule.apply_rule(protocol, hostname, original_object_name,
3621                      &s->redirect, &redirect_code);
3622     // Apply a custom HTTP response code
3623     if (redirect_code > 0)
3624       s->err.http_ret = redirect_code; // Apply a custom HTTP response code
3625     ldout(s->cct, 10) << "error handler redirect code=" << redirect_code
3626                       << " proto+host:" << protocol << "://" << hostname
3627                       << " -> " << s->redirect << dendl;
3628     return -ERR_WEBSITE_REDIRECT;
3629   } else if (err_no == -ERR_WEBSITE_REDIRECT) {
3630     // Do nothing here, this redirect will be handled in abort_early's ERR_WEBSITE_REDIRECT block
3631     // Do NOT fire the ErrorDoc handler
3632   } else if (!s->bucket_info.website_conf.error_doc.empty()) {
3633     /* This serves an entire page!
3634        On success, it will return zero, and no further content should be sent to the socket
3635        On failure, we need the double-error handler
3636      */
3637     new_err_no = RGWHandler_REST_S3Website::serve_errordoc(http_error_code, s->bucket_info.website_conf.error_doc);
3638     if (new_err_no && new_err_no != -1) {
3639       err_no = new_err_no;
3640     }
3641   } else {
3642     ldout(s->cct, 20) << "No special error handling today!" << dendl;
3643   }
3644
3645   return err_no;
3646 }
3647
3648 RGWOp* RGWHandler_REST_Obj_S3Website::get_obj_op(bool get_data)
3649 {
3650   /** If we are in website mode, then it is explicitly impossible to run GET or
3651    * HEAD on the actual directory. We must convert the request to run on the
3652    * suffix object instead!
3653    */
3654   RGWGetObj_ObjStore_S3Website* op = new RGWGetObj_ObjStore_S3Website;
3655   op->set_get_data(get_data);
3656   return op;
3657 }
3658
3659 RGWOp* RGWHandler_REST_Bucket_S3Website::get_obj_op(bool get_data)
3660 {
3661   /** If we are in website mode, then it is explicitly impossible to run GET or
3662    * HEAD on the actual directory. We must convert the request to run on the
3663    * suffix object instead!
3664    */
3665   RGWGetObj_ObjStore_S3Website* op = new RGWGetObj_ObjStore_S3Website;
3666   op->set_get_data(get_data);
3667   return op;
3668 }
3669
3670 RGWOp* RGWHandler_REST_Service_S3Website::get_obj_op(bool get_data)
3671 {
3672   /** If we are in website mode, then it is explicitly impossible to run GET or
3673    * HEAD on the actual directory. We must convert the request to run on the
3674    * suffix object instead!
3675    */
3676   RGWGetObj_ObjStore_S3Website* op = new RGWGetObj_ObjStore_S3Website;
3677   op->set_get_data(get_data);
3678   return op;
3679 }
3680
3681
3682 namespace rgw {
3683 namespace auth {
3684 namespace s3 {
3685
3686 static rgw::auth::Completer::cmplptr_t
3687 null_completer_factory(const boost::optional<std::string>& secret_key)
3688 {
3689   return nullptr;
3690 }
3691
3692
3693 AWSEngine::VersionAbstractor::auth_data_t
3694 AWSGeneralAbstractor::get_auth_data(const req_state* const s) const
3695 {
3696   AwsVersion version;
3697   AwsRoute route;
3698   std::tie(version, route) = discover_aws_flavour(s->info);
3699
3700   if (version == AwsVersion::V2) {
3701     return get_auth_data_v2(s);
3702   } else if (version == AwsVersion::V4) {
3703     return get_auth_data_v4(s, route == AwsRoute::QUERY_STRING);
3704   } else {
3705     /* FIXME(rzarzynski): handle anon user. */
3706     throw -EINVAL;
3707   }
3708 }
3709
3710 boost::optional<std::string>
3711 AWSGeneralAbstractor::get_v4_canonical_headers(
3712   const req_info& info,
3713   const boost::string_view& signedheaders,
3714   const bool using_qs) const
3715 {
3716   return rgw::auth::s3::get_v4_canonical_headers(info, signedheaders,
3717                                                  using_qs, false);
3718 }
3719
3720 AWSEngine::VersionAbstractor::auth_data_t
3721 AWSGeneralAbstractor::get_auth_data_v4(const req_state* const s,
3722                                        /* FIXME: const. */
3723                                        bool using_qs) const
3724 {
3725   boost::string_view access_key_id;
3726   boost::string_view signed_hdrs;
3727
3728   boost::string_view date;
3729   boost::string_view credential_scope;
3730   boost::string_view client_signature;
3731
3732   int ret = rgw::auth::s3::parse_credentials(s->info,
3733                                              access_key_id,
3734                                              credential_scope,
3735                                              signed_hdrs,
3736                                              client_signature,
3737                                              date,
3738                                              using_qs);
3739   if (ret < 0) {
3740     throw ret;
3741   }
3742
3743   /* craft canonical headers */
3744   boost::optional<std::string> canonical_headers = \
3745     get_v4_canonical_headers(s->info, signed_hdrs, using_qs);
3746   if (canonical_headers) {
3747     ldout(s->cct, 10) << "canonical headers format = " << *canonical_headers
3748                       << dendl;
3749   } else {
3750     throw -EPERM;
3751   }
3752
3753   /* Get the expected hash. */
3754   auto exp_payload_hash = rgw::auth::s3::get_v4_exp_payload_hash(s->info);
3755
3756   /* Craft canonical URI. Using std::move later so let it be non-const. */
3757   auto canonical_uri = rgw::auth::s3::get_v4_canonical_uri(s->info);
3758
3759   /* Craft canonical query string. std::moving later so non-const here. */
3760   auto canonical_qs = rgw::auth::s3::get_v4_canonical_qs(s->info, using_qs);
3761
3762   /* Craft canonical request. */
3763   auto canonical_req_hash = \
3764     rgw::auth::s3::get_v4_canon_req_hash(s->cct,
3765                                          s->info.method,
3766                                          std::move(canonical_uri),
3767                                          std::move(canonical_qs),
3768                                          std::move(*canonical_headers),
3769                                          signed_hdrs,
3770                                          exp_payload_hash);
3771
3772   auto string_to_sign = \
3773     rgw::auth::s3::get_v4_string_to_sign(s->cct,
3774                                          AWS4_HMAC_SHA256_STR,
3775                                          date,
3776                                          credential_scope,
3777                                          std::move(canonical_req_hash));
3778
3779   const auto sig_factory = std::bind(rgw::auth::s3::get_v4_signature,
3780                                      credential_scope,
3781                                      std::placeholders::_1,
3782                                      std::placeholders::_2,
3783                                      std::placeholders::_3);
3784
3785   /* Requests authenticated with the Query Parameters are treated as unsigned.
3786    * From "Authenticating Requests: Using Query Parameters (AWS Signature
3787    * Version 4)":
3788    *
3789    *   You don't include a payload hash in the Canonical Request, because
3790    *   when you create a presigned URL, you don't know the payload content
3791    *   because the URL is used to upload an arbitrary payload. Instead, you
3792    *   use a constant string UNSIGNED-PAYLOAD.
3793    *
3794    * This means we have absolutely no business in spawning completer. Both
3795    * aws4_auth_needs_complete and aws4_auth_streaming_mode are set to false
3796    * by default. We don't need to change that. */
3797   if (is_v4_payload_unsigned(exp_payload_hash) || is_v4_payload_empty(s)) {
3798     return {
3799       access_key_id,
3800       client_signature,
3801       std::move(string_to_sign),
3802       sig_factory,
3803       null_completer_factory
3804     };
3805   } else {
3806     /* We're going to handle a signed payload. Be aware that even empty HTTP
3807      * body (no payload) requires verification:
3808      *
3809      *   The x-amz-content-sha256 header is required for all AWS Signature
3810      *   Version 4 requests. It provides a hash of the request payload. If
3811      *   there is no payload, you must provide the hash of an empty string. */
3812     if (!is_v4_payload_streamed(exp_payload_hash)) {
3813       ldout(s->cct, 10) << "delaying v4 auth" << dendl;
3814
3815       /* payload in a single chunk */
3816       switch (s->op_type)
3817       {
3818         case RGW_OP_CREATE_BUCKET:
3819         case RGW_OP_PUT_OBJ:
3820         case RGW_OP_PUT_ACLS:
3821         case RGW_OP_PUT_CORS:
3822         case RGW_OP_INIT_MULTIPART: // in case that Init Multipart uses CHUNK encoding
3823         case RGW_OP_COMPLETE_MULTIPART:
3824         case RGW_OP_SET_BUCKET_VERSIONING:
3825         case RGW_OP_DELETE_MULTI_OBJ:
3826         case RGW_OP_ADMIN_SET_METADATA:
3827         case RGW_OP_SET_BUCKET_WEBSITE:
3828         case RGW_OP_PUT_BUCKET_POLICY:
3829         case RGW_OP_PUT_OBJ_TAGGING:
3830         case RGW_OP_PUT_LC:
3831         case RGW_OP_SET_REQUEST_PAYMENT:
3832           break;
3833         default:
3834           dout(10) << "ERROR: AWS4 completion for this operation NOT IMPLEMENTED" << dendl;
3835           throw -ERR_NOT_IMPLEMENTED;
3836       }
3837
3838       const auto cmpl_factory = std::bind(AWSv4ComplSingle::create,
3839                                           s,
3840                                           std::placeholders::_1);
3841       return {
3842         access_key_id,
3843         client_signature,
3844         std::move(string_to_sign),
3845         sig_factory,
3846         cmpl_factory
3847       };
3848     } else {
3849       /* IMHO "streamed" doesn't fit too good here. I would prefer to call
3850        * it "chunked" but let's be coherent with Amazon's terminology. */
3851
3852       dout(10) << "body content detected in multiple chunks" << dendl;
3853
3854       /* payload in multiple chunks */
3855
3856       switch(s->op_type)
3857       {
3858         case RGW_OP_PUT_OBJ:
3859           break;
3860         default:
3861           dout(10) << "ERROR: AWS4 completion for this operation NOT IMPLEMENTED (streaming mode)" << dendl;
3862           throw -ERR_NOT_IMPLEMENTED;
3863       }
3864
3865       dout(10) << "aws4 seed signature ok... delaying v4 auth" << dendl;
3866
3867       /* In the case of streamed payload client sets the x-amz-content-sha256
3868        * to "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" but uses "UNSIGNED-PAYLOAD"
3869        * when constructing the Canonical Request. */
3870
3871       /* In the case of single-chunk upload client set the header's value is
3872        * coherent with the one used for Canonical Request crafting. */
3873
3874       /* In the case of query string-based authentication there should be no
3875        * x-amz-content-sha256 header and the value "UNSIGNED-PAYLOAD" is used
3876        * for CanonReq. */
3877       const auto cmpl_factory = std::bind(AWSv4ComplMulti::create,
3878                                           s,
3879                                           date,
3880                                           credential_scope,
3881                                           client_signature,
3882                                           std::placeholders::_1);
3883       return {
3884         access_key_id,
3885         client_signature,
3886         std::move(string_to_sign),
3887         sig_factory,
3888         cmpl_factory
3889       };
3890     }
3891   }
3892 }
3893
3894
3895 boost::optional<std::string>
3896 AWSGeneralBoto2Abstractor::get_v4_canonical_headers(
3897   const req_info& info,
3898   const boost::string_view& signedheaders,
3899   const bool using_qs) const
3900 {
3901   return rgw::auth::s3::get_v4_canonical_headers(info, signedheaders,
3902                                                  using_qs, true);
3903 }
3904
3905
3906 AWSEngine::VersionAbstractor::auth_data_t
3907 AWSGeneralAbstractor::get_auth_data_v2(const req_state* const s) const
3908 {
3909   boost::string_view access_key_id;
3910   boost::string_view signature;
3911   bool qsr = false;
3912
3913   const char* http_auth = s->info.env->get("HTTP_AUTHORIZATION");
3914   if (! http_auth || http_auth[0] == '\0') {
3915     /* Credentials are provided in query string. We also need to verify
3916      * the "Expires" parameter now. */
3917     access_key_id = s->info.args.get("AWSAccessKeyId");
3918     signature = s->info.args.get("Signature");
3919     qsr = true;
3920
3921     boost::string_view expires = s->info.args.get("Expires");
3922     if (expires.empty()) {
3923       throw -EPERM;
3924     }
3925
3926     /* It looks we have the guarantee that expires is a null-terminated,
3927      * and thus string_view::data() can be safely used. */
3928     const time_t exp = atoll(expires.data());
3929     time_t now;
3930     time(&now);
3931
3932     if (now >= exp) {
3933       throw -EPERM;
3934     }
3935   } else {
3936     /* The "Authorization" HTTP header is being used. */
3937     const boost::string_view auth_str(http_auth + strlen("AWS "));
3938     const size_t pos = auth_str.rfind(':');
3939     if (pos != boost::string_view::npos) {
3940       access_key_id = auth_str.substr(0, pos);
3941       signature = auth_str.substr(pos + 1);
3942     }
3943   }
3944
3945   /* Let's canonize the HTTP headers that are covered by the AWS auth v2. */
3946   std::string string_to_sign;
3947   utime_t header_time;
3948   if (! rgw_create_s3_canonical_header(s->info, &header_time, string_to_sign,
3949         qsr)) {
3950     ldout(cct, 10) << "failed to create the canonized auth header\n"
3951                    << rgw::crypt_sanitize::auth{s,string_to_sign} << dendl;
3952     throw -EPERM;
3953   }
3954
3955   ldout(cct, 10) << "string_to_sign:\n"
3956                  << rgw::crypt_sanitize::auth{s,string_to_sign} << dendl;
3957
3958   if (!qsr && !is_time_skew_ok(header_time)) {
3959     throw -ERR_REQUEST_TIME_SKEWED;
3960   }
3961
3962   return {
3963     std::move(access_key_id),
3964     std::move(signature),
3965     std::move(string_to_sign),
3966     rgw::auth::s3::get_v2_signature,
3967     null_completer_factory
3968   };
3969 }
3970
3971
3972 AWSEngine::VersionAbstractor::auth_data_t
3973 AWSBrowserUploadAbstractor::get_auth_data_v2(const req_state* const s) const
3974 {
3975   return {
3976     s->auth.s3_postobj_creds.access_key,
3977     s->auth.s3_postobj_creds.signature,
3978     s->auth.s3_postobj_creds.encoded_policy.to_str(),
3979     rgw::auth::s3::get_v2_signature,
3980     null_completer_factory
3981   };
3982 }
3983
3984 AWSEngine::VersionAbstractor::auth_data_t
3985 AWSBrowserUploadAbstractor::get_auth_data_v4(const req_state* const s) const
3986 {
3987   const boost::string_view credential = s->auth.s3_postobj_creds.x_amz_credential;
3988
3989   /* grab access key id */
3990   const size_t pos = credential.find("/");
3991   const boost::string_view access_key_id = credential.substr(0, pos);
3992   dout(10) << "access key id = " << access_key_id << dendl;
3993
3994   /* grab credential scope */
3995   const boost::string_view credential_scope = credential.substr(pos + 1);
3996   dout(10) << "credential scope = " << credential_scope << dendl;
3997
3998   const auto sig_factory = std::bind(rgw::auth::s3::get_v4_signature,
3999                                      credential_scope,
4000                                      std::placeholders::_1,
4001                                      std::placeholders::_2,
4002                                      std::placeholders::_3);
4003
4004   return {
4005     access_key_id,
4006     s->auth.s3_postobj_creds.signature,
4007     s->auth.s3_postobj_creds.encoded_policy.to_str(),
4008     sig_factory,
4009     null_completer_factory
4010   };
4011 }
4012
4013 AWSEngine::VersionAbstractor::auth_data_t
4014 AWSBrowserUploadAbstractor::get_auth_data(const req_state* const s) const
4015 {
4016   if (s->auth.s3_postobj_creds.x_amz_algorithm == AWS4_HMAC_SHA256_STR) {
4017     ldout(s->cct, 0) << "Signature verification algorithm AWS v4"
4018                      << " (AWS4-HMAC-SHA256)" << dendl;
4019     return get_auth_data_v4(s);
4020   } else {
4021     ldout(s->cct, 0) << "Signature verification algorithm AWS v2" << dendl;
4022     return get_auth_data_v2(s);
4023   }
4024 }
4025
4026
4027 AWSEngine::result_t
4028 AWSEngine::authenticate(const req_state* const s) const
4029 {
4030   /* Small reminder: an ver_abstractor is allowed to throw! */
4031   const auto auth_data = ver_abstractor.get_auth_data(s);
4032
4033   if (auth_data.access_key_id.empty() || auth_data.client_signature.empty()) {
4034     return result_t::deny(-EINVAL);
4035   } else {
4036     return authenticate(auth_data.access_key_id,
4037                         auth_data.client_signature,
4038                         auth_data.string_to_sign,
4039                         auth_data.signature_factory,
4040                         auth_data.completer_factory,
4041                         s);
4042   }
4043 }
4044
4045 } /* namespace s3 */
4046 } /* namespace auth */
4047 } /* namespace rgw */
4048
4049 rgw::LDAPHelper* rgw::auth::s3::LDAPEngine::ldh = nullptr;
4050 std::mutex rgw::auth::s3::LDAPEngine::mtx;
4051
4052 void rgw::auth::s3::LDAPEngine::init(CephContext* const cct)
4053 {
4054   if (! ldh) {
4055     std::lock_guard<std::mutex> lck(mtx);
4056     if (! ldh) {
4057       const string& ldap_uri = cct->_conf->rgw_ldap_uri;
4058       const string& ldap_binddn = cct->_conf->rgw_ldap_binddn;
4059       const string& ldap_searchdn = cct->_conf->rgw_ldap_searchdn;
4060       const string& ldap_searchfilter = cct->_conf->rgw_ldap_searchfilter;
4061       const string& ldap_dnattr = cct->_conf->rgw_ldap_dnattr;
4062       std::string ldap_bindpw = parse_rgw_ldap_bindpw(cct);
4063
4064       ldh = new rgw::LDAPHelper(ldap_uri, ldap_binddn, ldap_bindpw,
4065                                 ldap_searchdn, ldap_searchfilter, ldap_dnattr);
4066
4067       ldh->init();
4068       ldh->bind();
4069     }
4070   }
4071 }
4072
4073 rgw::auth::RemoteApplier::acl_strategy_t
4074 rgw::auth::s3::LDAPEngine::get_acl_strategy() const
4075 {
4076   //This is based on the assumption that the default acl strategy in
4077   // get_perms_from_aclspec, will take care. Extra acl spec is not required.
4078   return nullptr;
4079 }
4080
4081 rgw::auth::RemoteApplier::AuthInfo
4082 rgw::auth::s3::LDAPEngine::get_creds_info(const rgw::RGWToken& token) const noexcept
4083 {
4084   /* The short form of "using" can't be used here -- we're aliasing a class'
4085    * member. */
4086   using acct_privilege_t = \
4087     rgw::auth::RemoteApplier::AuthInfo::acct_privilege_t;
4088
4089   return rgw::auth::RemoteApplier::AuthInfo {
4090     rgw_user(token.id),
4091     token.id,
4092     RGW_PERM_FULL_CONTROL,
4093     acct_privilege_t::IS_PLAIN_ACCT,
4094     TYPE_LDAP
4095   };
4096 }
4097
4098 rgw::auth::Engine::result_t
4099 rgw::auth::s3::LDAPEngine::authenticate(
4100   const boost::string_view& access_key_id,
4101   const boost::string_view& signature,
4102   const string_to_sign_t& string_to_sign,
4103   const signature_factory_t&,
4104   const completer_factory_t& completer_factory,
4105   const req_state* const s) const
4106 {
4107   /* boost filters and/or string_ref may throw on invalid input */
4108   rgw::RGWToken base64_token;
4109   try {
4110     base64_token = rgw::from_base64(access_key_id);
4111   } catch (...) {
4112     base64_token = std::string("");
4113   }
4114
4115   if (! base64_token.valid()) {
4116     return result_t::deny();
4117   }
4118
4119   //TODO: Uncomment, when we have a migration plan in place.
4120   //Check if a user of type other than 'ldap' is already present, if yes, then
4121   //return error.
4122   /*RGWUserInfo user_info;
4123   user_info.user_id = base64_token.id;
4124   if (rgw_get_user_info_by_uid(store, user_info.user_id, user_info) >= 0) {
4125     if (user_info.type != TYPE_LDAP) {
4126       ldout(cct, 10) << "ERROR: User id of type: " << user_info.type << " is already present" << dendl;
4127       return nullptr;
4128     }
4129   }*/
4130
4131   if (ldh->auth(base64_token.id, base64_token.key) != 0) {
4132     return result_t::deny();
4133   }
4134
4135   auto apl = apl_factory->create_apl_remote(cct, s, get_acl_strategy(),
4136                                             get_creds_info(base64_token));
4137   return result_t::grant(std::move(apl), completer_factory(boost::none));
4138 }
4139
4140
4141 /* LocalEndgine */
4142 rgw::auth::Engine::result_t
4143 rgw::auth::s3::LocalEngine::authenticate(
4144   const boost::string_view& _access_key_id,
4145   const boost::string_view& signature,
4146   const string_to_sign_t& string_to_sign,
4147   const signature_factory_t& signature_factory,
4148   const completer_factory_t& completer_factory,
4149   const req_state* const s) const
4150 {
4151   /* get the user info */
4152   RGWUserInfo user_info;
4153   /* TODO(rzarzynski): we need to have string-view taking variant. */
4154   const std::string access_key_id = _access_key_id.to_string();
4155   if (rgw_get_user_info_by_access_key(store, access_key_id, user_info) < 0) {
4156       ldout(cct, 5) << "error reading user info, uid=" << access_key_id
4157               << " can't authenticate" << dendl;
4158       return result_t::deny(-ERR_INVALID_ACCESS_KEY);
4159   }
4160   //TODO: Uncomment, when we have a migration plan in place.
4161   /*else {
4162     if (s->user->type != TYPE_RGW) {
4163       ldout(cct, 10) << "ERROR: User id of type: " << s->user->type
4164                      << " is present" << dendl;
4165       throw -EPERM;
4166     }
4167   }*/
4168
4169   const auto iter = user_info.access_keys.find(access_key_id);
4170   if (iter == std::end(user_info.access_keys)) {
4171     ldout(cct, 0) << "ERROR: access key not encoded in user info" << dendl;
4172     return result_t::deny(-EPERM);
4173   }
4174   const RGWAccessKey& k = iter->second;
4175
4176   const VersionAbstractor::server_signature_t server_signature = \
4177     signature_factory(cct, k.key, string_to_sign);
4178
4179   ldout(cct, 15) << "string_to_sign="
4180                  << rgw::crypt_sanitize::log_content{string_to_sign}
4181                  << dendl;
4182   ldout(cct, 15) << "server signature=" << server_signature << dendl;
4183   ldout(cct, 15) << "client signature=" << signature << dendl;
4184   ldout(cct, 15) << "compare=" << signature.compare(server_signature) << dendl;
4185
4186   if (static_cast<boost::string_view>(server_signature) != signature) {
4187     return result_t::deny(-ERR_SIGNATURE_NO_MATCH);
4188   }
4189
4190   auto apl = apl_factory->create_apl_local(cct, s, user_info, k.subuser);
4191   return result_t::grant(std::move(apl), completer_factory(k.key));
4192 }
4193
4194 bool rgw::auth::s3::S3AnonymousEngine::is_applicable(
4195   const req_state* s
4196 ) const noexcept {
4197   if (s->op == OP_OPTIONS) {
4198     return true;
4199   }
4200
4201   AwsVersion version;
4202   AwsRoute route;
4203   std::tie(version, route) = discover_aws_flavour(s->info);
4204
4205   return route == AwsRoute::QUERY_STRING && version == AwsVersion::UNKOWN;
4206 }