ceph/src/rgw/rgw_common.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3
   4 #include <errno.h>
   5 #include <vector>
   6 #include <algorithm>
   7 #include <string>
   8 #include <boost/tokenizer.hpp>
   9 #include <boost/algorithm/string.hpp>
  10 #include <boost/utility/string_view.hpp>
  11
  12 #include "json_spirit/json_spirit.h"
  13 #include "common/ceph_json.h"
  14
  15 #include "rgw_op.h"
  16 #include "rgw_common.h"
  17 #include "rgw_acl.h"
  18 #include "rgw_string.h"
  19 #include "rgw_rados.h"
  20 #include "rgw_http_errors.h"
  21
  22 #include "common/ceph_crypto.h"
  23 #include "common/armor.h"
  24 #include "common/errno.h"
  25 #include "common/Clock.h"
  26 #include "common/Formatter.h"
  27 #include "common/perf_counters.h"
  28 #include "common/strtol.h"
  29 #include "include/str_list.h"
  30 #include "auth/Crypto.h"
  31 #include "rgw_crypt_sanitize.h"
  32
  33 #include <sstream>
  34
  35 #define dout_context g_ceph_context
  36 #define dout_subsys ceph_subsys_rgw
  37
  38 using boost::none;
  39 using boost::optional;
  40
  41 using rgw::IAM::ARN;
  42 using rgw::IAM::Effect;
  43 using rgw::IAM::op_to_perm;
  44 using rgw::IAM::Policy;
  45
  46 PerfCounters *perfcounter = NULL;
  47
  48 const uint32_t RGWBucketInfo::NUM_SHARDS_BLIND_BUCKET(UINT32_MAX);
  49
  50 rgw_http_errors rgw_http_s3_errors({
  51     { 0, {200, "" }},
  52     { STATUS_CREATED, {201, "Created" }},
  53     { STATUS_ACCEPTED, {202, "Accepted" }},
  54     { STATUS_NO_CONTENT, {204, "NoContent" }},
  55     { STATUS_PARTIAL_CONTENT, {206, "" }},
  56     { ERR_PERMANENT_REDIRECT, {301, "PermanentRedirect" }},
  57     { ERR_WEBSITE_REDIRECT, {301, "WebsiteRedirect" }},
  58     { STATUS_REDIRECT, {303, "" }},
  59     { ERR_NOT_MODIFIED, {304, "NotModified" }},
  60     { EINVAL, {400, "InvalidArgument" }},
  61     { ERR_INVALID_REQUEST, {400, "InvalidRequest" }},
  62     { ERR_INVALID_DIGEST, {400, "InvalidDigest" }},
  63     { ERR_BAD_DIGEST, {400, "BadDigest" }},
  64     { ERR_INVALID_LOCATION_CONSTRAINT, {400, "InvalidLocationConstraint" }},
  65     { ERR_ZONEGROUP_DEFAULT_PLACEMENT_MISCONFIGURATION, {400, "ZonegroupDefaultPlacementMisconfiguration" }},
  66     { ERR_INVALID_BUCKET_NAME, {400, "InvalidBucketName" }},
  67     { ERR_INVALID_OBJECT_NAME, {400, "InvalidObjectName" }},
  68     { ERR_UNRESOLVABLE_EMAIL, {400, "UnresolvableGrantByEmailAddress" }},
  69     { ERR_INVALID_PART, {400, "InvalidPart" }},
  70     { ERR_INVALID_PART_ORDER, {400, "InvalidPartOrder" }},
  71     { ERR_REQUEST_TIMEOUT, {400, "RequestTimeout" }},
  72     { ERR_TOO_LARGE, {400, "EntityTooLarge" }},
  73     { ERR_TOO_SMALL, {400, "EntityTooSmall" }},
  74     { ERR_TOO_MANY_BUCKETS, {400, "TooManyBuckets" }},
  75     { ERR_MALFORMED_XML, {400, "MalformedXML" }},
  76     { ERR_AMZ_CONTENT_SHA256_MISMATCH, {400, "XAmzContentSHA256Mismatch" }},
  77     { ERR_INVALID_TAG, {400, "InvalidTag"}},
  78     { ERR_MALFORMED_ACL_ERROR, {400, "MalformedACLError" }},
  79     { ERR_LENGTH_REQUIRED, {411, "MissingContentLength" }},
  80     { EACCES, {403, "AccessDenied" }},
  81     { EPERM, {403, "AccessDenied" }},
  82     { ERR_SIGNATURE_NO_MATCH, {403, "SignatureDoesNotMatch" }},
  83     { ERR_INVALID_ACCESS_KEY, {403, "InvalidAccessKeyId" }},
  84     { ERR_USER_SUSPENDED, {403, "UserSuspended" }},
  85     { ERR_REQUEST_TIME_SKEWED, {403, "RequestTimeTooSkewed" }},
  86     { ERR_QUOTA_EXCEEDED, {403, "QuotaExceeded" }},
  87     { ENOENT, {404, "NoSuchKey" }},
  88     { ERR_NO_SUCH_BUCKET, {404, "NoSuchBucket" }},
  89     { ERR_NO_SUCH_WEBSITE_CONFIGURATION, {404, "NoSuchWebsiteConfiguration" }},
  90     { ERR_NO_SUCH_UPLOAD, {404, "NoSuchUpload" }},
  91     { ERR_NOT_FOUND, {404, "Not Found"}},
  92     { ERR_NO_SUCH_LC, {404, "NoSuchLifecycleConfiguration"}},
  93     { ERR_NO_SUCH_BUCKET_POLICY, {404, "NoSuchBucketPolicy"}},
  94     { ERR_NO_SUCH_USER, {404, "NoSuchUser"}},
  95     { ERR_NO_SUCH_SUBUSER, {404, "NoSuchSubUser"}},
  96     { ERR_METHOD_NOT_ALLOWED, {405, "MethodNotAllowed" }},
  97     { ETIMEDOUT, {408, "RequestTimeout" }},
  98     { EEXIST, {409, "BucketAlreadyExists" }},
  99     { ERR_USER_EXIST, {409, "UserAlreadyExists" }},
 100     { ERR_EMAIL_EXIST, {409, "EmailExists" }},
 101     { ERR_KEY_EXIST, {409, "KeyExists"}},
 102     { ERR_TAG_CONFLICT, {409, "OperationAborted"}},
 103     { ERR_INVALID_SECRET_KEY, {400, "InvalidSecretKey"}},
 104     { ERR_INVALID_KEY_TYPE, {400, "InvalidKeyType"}},
 105     { ERR_INVALID_CAP, {400, "InvalidCapability"}},
 106     { ERR_INVALID_TENANT_NAME, {400, "InvalidTenantName" }},
 107     { ENOTEMPTY, {409, "BucketNotEmpty" }},
 108     { ERR_PRECONDITION_FAILED, {412, "PreconditionFailed" }},
 109     { ERANGE, {416, "InvalidRange" }},
 110     { ERR_UNPROCESSABLE_ENTITY, {422, "UnprocessableEntity" }},
 111     { ERR_LOCKED, {423, "Locked" }},
 112     { ERR_INTERNAL_ERROR, {500, "InternalError" }},
 113     { ERR_NOT_IMPLEMENTED, {501, "NotImplemented" }},
 114     { ERR_SERVICE_UNAVAILABLE, {503, "ServiceUnavailable"}},
 115     { ERR_ZERO_IN_URL, {400, "InvalidRequest" }},
 116 });
 117
 118 rgw_http_errors rgw_http_swift_errors({
 119     { EACCES, {403, "AccessDenied" }},
 120     { EPERM, {401, "AccessDenied" }},
 121     { ERR_USER_SUSPENDED, {401, "UserSuspended" }},
 122     { ERR_INVALID_UTF8, {412, "Invalid UTF8" }},
 123     { ERR_BAD_URL, {412, "Bad URL" }},
 124     { ERR_NOT_SLO_MANIFEST, {400, "Not an SLO manifest" }},
 125     { ERR_QUOTA_EXCEEDED, {413, "QuotaExceeded" }},
 126     /* FIXME(rzarzynski): we need to find a way to apply Swift's error handling
 127      * procedures also for ERR_ZERO_IN_URL. This make a problem as the validation
 128      * is performed very early, even before setting the req_state::proto_flags. */
 129     { ERR_ZERO_IN_URL, {412, "Invalid UTF8 or contains NULL"}},
 130 });
 131
 132 int rgw_perf_start(CephContext *cct)
 133 {
 134   PerfCountersBuilder plb(cct, cct->_conf->name.to_str(), l_rgw_first, l_rgw_last);
 135
 136   plb.add_u64_counter(l_rgw_req, "req", "Requests");
 137   plb.add_u64_counter(l_rgw_failed_req, "failed_req", "Aborted requests");
 138
 139   plb.add_u64_counter(l_rgw_get, "get", "Gets");
 140   plb.add_u64_counter(l_rgw_get_b, "get_b", "Size of gets");
 141   plb.add_time_avg(l_rgw_get_lat, "get_initial_lat", "Get latency");
 142   plb.add_u64_counter(l_rgw_put, "put", "Puts");
 143   plb.add_u64_counter(l_rgw_put_b, "put_b", "Size of puts");
 144   plb.add_time_avg(l_rgw_put_lat, "put_initial_lat", "Put latency");
 145
 146   plb.add_u64(l_rgw_qlen, "qlen", "Queue length");
 147   plb.add_u64(l_rgw_qactive, "qactive", "Active requests queue");
 148
 149   plb.add_u64_counter(l_rgw_cache_hit, "cache_hit", "Cache hits");
 150   plb.add_u64_counter(l_rgw_cache_miss, "cache_miss", "Cache miss");
 151
 152   plb.add_u64_counter(l_rgw_keystone_token_cache_hit, "keystone_token_cache_hit", "Keystone token cache hits");
 153   plb.add_u64_counter(l_rgw_keystone_token_cache_miss, "keystone_token_cache_miss", "Keystone token cache miss");
 154
 155   perfcounter = plb.create_perf_counters();
 156   cct->get_perfcounters_collection()->add(perfcounter);
 157   return 0;
 158 }
 159
 160 void rgw_perf_stop(CephContext *cct)
 161 {
 162   assert(perfcounter);
 163   cct->get_perfcounters_collection()->remove(perfcounter);
 164   delete perfcounter;
 165 }
 166
 167 using namespace ceph::crypto;
 168
 169 rgw_err::
 170 rgw_err()
 171 {
 172   clear();
 173 }
 174
 175 void rgw_err::
 176 clear()
 177 {
 178   http_ret = 200;
 179   ret = 0;
 180   err_code.clear();
 181 }
 182
 183 bool rgw_err::
 184 is_clear() const
 185 {
 186   return (http_ret == 200);
 187 }
 188
 189 bool rgw_err::
 190 is_err() const
 191 {
 192   return !(http_ret >= 200 && http_ret <= 399);
 193 }
 194
 195 // The requestURI transferred from the frontend can be abs_path or absoluteURI
 196 // If it is absoluteURI, we should adjust it to abs_path for the following
 197 // S3 authorization and some other processes depending on the requestURI
 198 // The absoluteURI can start with "http://", "https://", "ws://" or "wss://"
 199 static string get_abs_path(const string& request_uri) {
 200   const static string ABS_PREFIXS[] = {"http://", "https://", "ws://", "wss://"};
 201   bool isAbs = false;
 202   for (int i = 0; i < 4; ++i) {
 203     if (boost::algorithm::starts_with(request_uri, ABS_PREFIXS[i])) {
 204       isAbs = true;
 205       break;
 206     }
 207   }
 208   if (!isAbs) {  // it is not a valid absolute uri
 209     return request_uri;
 210   }
 211   size_t beg_pos = request_uri.find("://") + 3;
 212   size_t len = request_uri.size();
 213   beg_pos = request_uri.find('/', beg_pos);
 214   if (beg_pos == string::npos) return request_uri;
 215   return request_uri.substr(beg_pos, len - beg_pos);
 216 }
 217
 218 req_info::req_info(CephContext *cct, const class RGWEnv *env) : env(env) {
 219   method = env->get("REQUEST_METHOD", "");
 220   script_uri = env->get("SCRIPT_URI", cct->_conf->rgw_script_uri.c_str());
 221   request_uri = env->get("REQUEST_URI", cct->_conf->rgw_request_uri.c_str());
 222   if (request_uri[0] != '/') {
 223     request_uri = get_abs_path(request_uri);
 224   }
 225   auto pos = request_uri.find('?');
 226   if (pos != string::npos) {
 227     request_params = request_uri.substr(pos + 1);
 228     request_uri = request_uri.substr(0, pos);
 229   } else {
 230     request_params = env->get("QUERY_STRING", "");
 231   }
 232   host = env->get("HTTP_HOST", "");
 233
 234   // strip off any trailing :port from host (added by CrossFTP and maybe others)
 235   size_t colon_offset = host.find_last_of(':');
 236   if (colon_offset != string::npos) {
 237     bool all_digits = true;
 238     for (unsigned i = colon_offset + 1; i < host.size(); ++i) {
 239       if (!isdigit(host[i])) {
 240         all_digits = false;
 241         break;
 242       }
 243     }
 244     if (all_digits) {
 245       host.resize(colon_offset);
 246     }
 247   }
 248 }
 249
 250 void req_info::rebuild_from(req_info& src)
 251 {
 252   method = src.method;
 253   script_uri = src.script_uri;
 254   args = src.args;
 255   if (src.effective_uri.empty()) {
 256     request_uri = src.request_uri;
 257   } else {
 258     request_uri = src.effective_uri;
 259   }
 260   effective_uri.clear();
 261   host = src.host;
 262
 263   x_meta_map = src.x_meta_map;
 264   x_meta_map.erase("x-amz-date");
 265 }
 266
 267
 268 req_state::req_state(CephContext* _cct, RGWEnv* e, RGWUserInfo* u)
 269   : cct(_cct), cio(NULL), op(OP_UNKNOWN), user(u), has_acl_header(false),
 270     info(_cct, e)
 271 {
 272   enable_ops_log = e->conf.enable_ops_log;
 273   enable_usage_log = e->conf.enable_usage_log;
 274   defer_to_bucket_acls = e->conf.defer_to_bucket_acls;
 275   content_started = false;
 276   format = 0;
 277   formatter = NULL;
 278   expect_cont = false;
 279
 280   obj_size = 0;
 281   prot_flags = 0;
 282
 283   system_request = false;
 284
 285   time = ceph_clock_now();
 286   perm_mask = 0;
 287   bucket_instance_shard_id = -1;
 288   content_length = 0;
 289   bucket_exists = false;
 290   has_bad_meta = false;
 291   length = NULL;
 292   local_source = false;
 293
 294   obj_ctx = NULL;
 295 }
 296
 297 req_state::~req_state() {
 298   delete formatter;
 299 }
 300
 301 bool search_err(rgw_http_errors& errs, int err_no, bool is_website_redirect, int& http_ret, string& code)
 302 {
 303   auto r = errs.find(err_no);
 304   if (r != errs.end()) {
 305     if (! is_website_redirect)
 306       http_ret = r->second.first;
 307      code = r->second.second;
 308      return true;
 309   }
 310   return false;
 311 }
 312
 313 void set_req_state_err(struct rgw_err& err,     /* out */
 314                         int err_no,             /* in  */
 315                         const int prot_flags)   /* in  */
 316 {
 317   if (err_no < 0)
 318     err_no = -err_no;
 319
 320   err.ret = -err_no;
 321   bool is_website_redirect = false;
 322
 323   if (prot_flags & RGW_REST_SWIFT) {
 324     if (search_err(rgw_http_swift_errors, err_no, is_website_redirect, err.http_ret, err.err_code))
 325       return;
 326   }
 327
 328   //Default to searching in s3 errors
 329   is_website_redirect |= (prot_flags & RGW_REST_WEBSITE)
 330                 && err_no == ERR_WEBSITE_REDIRECT && err.is_clear();
 331   if (search_err(rgw_http_s3_errors, err_no, is_website_redirect, err.http_ret, err.err_code))
 332       return;
 333   dout(0) << "WARNING: set_req_state_err err_no=" << err_no
 334         << " resorting to 500" << dendl;
 335
 336   err.http_ret = 500;
 337   err.err_code = "UnknownError";
 338 }
 339
 340 void set_req_state_err(struct req_state* s, int err_no, const string& err_msg)
 341 {
 342   if (s) {
 343     set_req_state_err(s, err_no);
 344     s->err.message = err_msg;
 345   }
 346 }
 347
 348 void set_req_state_err(struct req_state* s, int err_no)
 349 {
 350   if (s) {
 351     set_req_state_err(s->err, err_no, s->prot_flags);
 352   }
 353 }
 354
 355 void dump(struct req_state* s)
 356 {
 357   if (s->format != RGW_FORMAT_HTML)
 358     s->formatter->open_object_section("Error");
 359   if (!s->err.err_code.empty())
 360     s->formatter->dump_string("Code", s->err.err_code);
 361   if (!s->err.message.empty())
 362     s->formatter->dump_string("Message", s->err.message);
 363   if (!s->bucket_name.empty())  // TODO: connect to expose_bucket
 364     s->formatter->dump_string("BucketName", s->bucket_name);
 365   if (!s->trans_id.empty())     // TODO: connect to expose_bucket or another toggle
 366     s->formatter->dump_string("RequestId", s->trans_id);
 367   s->formatter->dump_string("HostId", s->host_id);
 368   if (s->format != RGW_FORMAT_HTML)
 369     s->formatter->close_section();
 370 }
 371
 372 struct str_len {
 373   const char *str;
 374   int len;
 375 };
 376
 377 #define STR_LEN_ENTRY(s) { s, sizeof(s) - 1 }
 378
 379 struct str_len meta_prefixes[] = { STR_LEN_ENTRY("HTTP_X_AMZ"),
 380                                    STR_LEN_ENTRY("HTTP_X_GOOG"),
 381                                    STR_LEN_ENTRY("HTTP_X_DHO"),
 382                                    STR_LEN_ENTRY("HTTP_X_RGW"),
 383                                    STR_LEN_ENTRY("HTTP_X_OBJECT"),
 384                                    STR_LEN_ENTRY("HTTP_X_CONTAINER"),
 385                                    STR_LEN_ENTRY("HTTP_X_ACCOUNT"),
 386                                    {NULL, 0} };
 387
 388
 389 void req_info::init_meta_info(bool *found_bad_meta)
 390 {
 391   x_meta_map.clear();
 392
 393   for (const auto& kv: env->get_map()) {
 394     const char *prefix;
 395     const string& header_name = kv.first;
 396     const string& val = kv.second;
 397     for (int prefix_num = 0; (prefix = meta_prefixes[prefix_num].str) != NULL; prefix_num++) {
 398       int len = meta_prefixes[prefix_num].len;
 399       const char *p = header_name.c_str();
 400       if (strncmp(p, prefix, len) == 0) {
 401         dout(10) << "meta>> " << p << dendl;
 402         const char *name = p+len; /* skip the prefix */
 403         int name_len = header_name.size() - len;
 404
 405         if (found_bad_meta && strncmp(name, "_META_", name_len) == 0)
 406           *found_bad_meta = true;
 407
 408         char name_low[meta_prefixes[0].len + name_len + 1];
 409         snprintf(name_low, meta_prefixes[0].len - 5 + name_len + 1, "%s%s", meta_prefixes[0].str + 5 /* skip HTTP_ */, name); // normalize meta prefix
 410         int j;
 411         for (j = 0; name_low[j]; j++) {
 412           if (name_low[j] != '_')
 413             name_low[j] = tolower(name_low[j]);
 414           else
 415             name_low[j] = '-';
 416         }
 417         name_low[j] = 0;
 418
 419         auto it = x_meta_map.find(name_low);
 420         if (it != x_meta_map.end()) {
 421           string old = it->second;
 422           boost::algorithm::trim_right(old);
 423           old.append(",");
 424           old.append(val);
 425           x_meta_map[name_low] = old;
 426         } else {
 427           x_meta_map[name_low] = val;
 428         }
 429       }
 430     }
 431   }
 432   for (const auto& kv: x_meta_map) {
 433     dout(10) << "x>> " << kv.first << ":" << rgw::crypt_sanitize::x_meta_map{kv.first, kv.second} << dendl;
 434   }
 435 }
 436
 437 std::ostream& operator<<(std::ostream& oss, const rgw_err &err)
 438 {
 439   oss << "rgw_err(http_ret=" << err.http_ret << ", err_code='" << err.err_code << "') ";
 440   return oss;
 441 }
 442
 443 string rgw_string_unquote(const string& s)
 444 {
 445   if (s[0] != '"' || s.size() < 2)
 446     return s;
 447
 448   int len;
 449   for (len = s.size(); len > 2; --len) {
 450     if (s[len - 1] != ' ')
 451       break;
 452   }
 453
 454   if (s[len-1] != '"')
 455     return s;
 456
 457   return s.substr(1, len - 2);
 458 }
 459
 460 static bool check_str_end(const char *s)
 461 {
 462   if (!s)
 463     return false;
 464
 465   while (*s) {
 466     if (!isspace(*s))
 467       return false;
 468     s++;
 469   }
 470   return true;
 471 }
 472
 473 static bool check_gmt_end(const char *s)
 474 {
 475   if (!s || !*s)
 476     return false;
 477
 478   while (isspace(*s)) {
 479     ++s;
 480   }
 481
 482   /* check for correct timezone */
 483   if ((strncmp(s, "GMT", 3) != 0) &&
 484       (strncmp(s, "UTC", 3) != 0)) {
 485     return false;
 486   }
 487
 488   return true;
 489 }
 490
 491 static bool parse_rfc850(const char *s, struct tm *t)
 492 {
 493   memset(t, 0, sizeof(*t));
 494   return check_gmt_end(strptime(s, "%A, %d-%b-%y %H:%M:%S ", t));
 495 }
 496
 497 static bool parse_asctime(const char *s, struct tm *t)
 498 {
 499   memset(t, 0, sizeof(*t));
 500   return check_str_end(strptime(s, "%a %b %d %H:%M:%S %Y", t));
 501 }
 502
 503 static bool parse_rfc1123(const char *s, struct tm *t)
 504 {
 505   memset(t, 0, sizeof(*t));
 506   return check_gmt_end(strptime(s, "%a, %d %b %Y %H:%M:%S ", t));
 507 }
 508
 509 static bool parse_rfc1123_alt(const char *s, struct tm *t)
 510 {
 511   memset(t, 0, sizeof(*t));
 512   return check_str_end(strptime(s, "%a, %d %b %Y %H:%M:%S %z", t));
 513 }
 514
 515 bool parse_rfc2616(const char *s, struct tm *t)
 516 {
 517   return parse_rfc850(s, t) || parse_asctime(s, t) || parse_rfc1123(s, t) || parse_rfc1123_alt(s,t);
 518 }
 519
 520 bool parse_iso8601(const char *s, struct tm *t, uint32_t *pns, bool extended_format)
 521 {
 522   memset(t, 0, sizeof(*t));
 523   const char *p;
 524
 525   if (!s)
 526     s = "";
 527
 528   if (extended_format) {
 529     p = strptime(s, "%Y-%m-%dT%T", t);
 530     if (!p) {
 531       p = strptime(s, "%Y-%m-%d %T", t);
 532     }
 533   } else {
 534     p = strptime(s, "%Y%m%dT%H%M%S", t);
 535   }
 536   if (!p) {
 537     dout(0) << "parse_iso8601 failed" << dendl;
 538     return false;
 539   }
 540   const boost::string_view str = rgw_trim_whitespace(boost::string_view(p));
 541   int len = str.size();
 542
 543   if (len == 0 || (len == 1 && str[0] == 'Z'))
 544     return true;
 545
 546   if (str[0] != '.' ||
 547       str[len - 1] != 'Z')
 548     return false;
 549
 550   uint32_t ms;
 551   boost::string_view nsstr = str.substr(1,  len - 2);
 552   int r = stringtoul(nsstr.to_string(), &ms);
 553   if (r < 0)
 554     return false;
 555
 556   if (!pns) {
 557     return true;
 558   }
 559
 560   if (nsstr.size() > 9) {
 561     nsstr = nsstr.substr(0, 9);
 562   }
 563
 564   uint64_t mul_table[] = { 0,
 565     100000000LL,
 566     10000000LL,
 567     1000000LL,
 568     100000LL,
 569     10000LL,
 570     1000LL,
 571     100LL,
 572     10LL,
 573     1 };
 574
 575
 576   *pns = ms * mul_table[nsstr.size()];
 577
 578   return true;
 579 }
 580
 581 int parse_key_value(string& in_str, const char *delim, string& key, string& val)
 582 {
 583   if (delim == NULL)
 584     return -EINVAL;
 585
 586   auto pos = in_str.find(delim);
 587   if (pos == string::npos)
 588     return -EINVAL;
 589
 590   key = rgw_trim_whitespace(in_str.substr(0, pos));
 591   val = rgw_trim_whitespace(in_str.substr(pos + 1));
 592
 593   return 0;
 594 }
 595
 596 int parse_key_value(string& in_str, string& key, string& val)
 597 {
 598   return parse_key_value(in_str, "=", key,val);
 599 }
 600
 601 boost::optional<std::pair<boost::string_view, boost::string_view>>
 602 parse_key_value(const boost::string_view& in_str,
 603                 const boost::string_view& delim)
 604 {
 605   const size_t pos = in_str.find(delim);
 606   if (pos == boost::string_view::npos) {
 607     return boost::none;
 608   }
 609
 610   const auto key = rgw_trim_whitespace(in_str.substr(0, pos));
 611   const auto val = rgw_trim_whitespace(in_str.substr(pos + 1));
 612
 613   return std::make_pair(key, val);
 614 }
 615
 616 boost::optional<std::pair<boost::string_view, boost::string_view>>
 617 parse_key_value(const boost::string_view& in_str)
 618 {
 619   return parse_key_value(in_str, "=");
 620 }
 621
 622 int parse_time(const char *time_str, real_time *time)
 623 {
 624   struct tm tm;
 625   uint32_t ns = 0;
 626
 627   if (!parse_rfc2616(time_str, &tm) && !parse_iso8601(time_str, &tm, &ns)) {
 628     return -EINVAL;
 629   }
 630
 631   time_t sec = internal_timegm(&tm);
 632   *time = utime_t(sec, ns).to_real_time();
 633
 634   return 0;
 635 }
 636
 637 #define TIME_BUF_SIZE 128
 638
 639 void rgw_to_iso8601(const real_time& t, char *dest, int buf_size)
 640 {
 641   utime_t ut(t);
 642
 643   char buf[TIME_BUF_SIZE];
 644   struct tm result;
 645   time_t epoch = ut.sec();
 646   struct tm *tmp = gmtime_r(&epoch, &result);
 647   if (tmp == NULL)
 648     return;
 649
 650   if (strftime(buf, sizeof(buf), "%Y-%m-%dT%T", tmp) == 0)
 651     return;
 652
 653   snprintf(dest, buf_size, "%s.%03dZ", buf, (int)(ut.usec() / 1000));
 654 }
 655
 656 void rgw_to_iso8601(const real_time& t, string *dest)
 657 {
 658   char buf[TIME_BUF_SIZE];
 659   rgw_to_iso8601(t, buf, sizeof(buf));
 660   *dest = buf;
 661 }
 662
 663
 664 string rgw_to_asctime(const utime_t& t)
 665 {
 666   stringstream s;
 667   t.asctime(s);
 668   return s.str();
 669 }
 670
 671 /*
 672  * calculate the sha1 value of a given msg and key
 673  */
 674 void calc_hmac_sha1(const char *key, int key_len,
 675                     const char *msg, int msg_len, char *dest)
 676 /* destination should be CEPH_CRYPTO_HMACSHA1_DIGESTSIZE bytes long */
 677 {
 678   HMACSHA1 hmac((const unsigned char *)key, key_len);
 679   hmac.Update((const unsigned char *)msg, msg_len);
 680   hmac.Final((unsigned char *)dest);
 681 }
 682
 683 /*
 684  * calculate the sha256 value of a given msg and key
 685  */
 686 void calc_hmac_sha256(const char *key, int key_len,
 687                       const char *msg, int msg_len, char *dest)
 688 {
 689   char hash_sha256[CEPH_CRYPTO_HMACSHA256_DIGESTSIZE];
 690
 691   HMACSHA256 hmac((const unsigned char *)key, key_len);
 692   hmac.Update((const unsigned char *)msg, msg_len);
 693   hmac.Final((unsigned char *)hash_sha256);
 694
 695   memcpy(dest, hash_sha256, CEPH_CRYPTO_HMACSHA256_DIGESTSIZE);
 696 }
 697
 698 using ceph::crypto::SHA256;
 699
 700 /*
 701  * calculate the sha256 hash value of a given msg
 702  */
 703 sha256_digest_t calc_hash_sha256(const boost::string_view& msg)
 704 {
 705   std::array<unsigned char, CEPH_CRYPTO_HMACSHA256_DIGESTSIZE> hash;
 706
 707   SHA256 hasher;
 708   hasher.Update(reinterpret_cast<const unsigned char*>(msg.data()), msg.size());
 709   hasher.Final(hash.data());
 710
 711   return hash;
 712 }
 713
 714 SHA256* calc_hash_sha256_open_stream()
 715 {
 716   return new SHA256;
 717 }
 718
 719 void calc_hash_sha256_update_stream(SHA256 *hash, const char *msg, int len)
 720 {
 721   hash->Update((const unsigned char *)msg, len);
 722 }
 723
 724 string calc_hash_sha256_close_stream(SHA256 **phash)
 725 {
 726   SHA256 *hash = *phash;
 727   if (!hash) {
 728     hash = calc_hash_sha256_open_stream();
 729   }
 730   char hash_sha256[CEPH_CRYPTO_HMACSHA256_DIGESTSIZE];
 731
 732   hash->Final((unsigned char *)hash_sha256);
 733
 734   char hex_str[(CEPH_CRYPTO_SHA256_DIGESTSIZE * 2) + 1];
 735   buf_to_hex((unsigned char *)hash_sha256, CEPH_CRYPTO_SHA256_DIGESTSIZE, hex_str);
 736
 737   delete hash;
 738   *phash = NULL;
 739
 740   return std::string(hex_str);
 741 }
 742
 743 std::string calc_hash_sha256_restart_stream(SHA256 **phash)
 744 {
 745   const auto hash = calc_hash_sha256_close_stream(phash);
 746   *phash = calc_hash_sha256_open_stream();
 747
 748   return hash;
 749 }
 750
 751 int gen_rand_base64(CephContext *cct, char *dest, int size) /* size should be the required string size + 1 */
 752 {
 753   char buf[size];
 754   char tmp_dest[size + 4]; /* so that there's space for the extra '=' characters, and some */
 755   int ret;
 756
 757   ret = get_random_bytes(buf, sizeof(buf));
 758   if (ret < 0) {
 759     lderr(cct) << "cannot get random bytes: " << cpp_strerror(-ret) << dendl;
 760     return ret;
 761   }
 762
 763   ret = ceph_armor(tmp_dest, &tmp_dest[sizeof(tmp_dest)],
 764                    (const char *)buf, ((const char *)buf) + ((size - 1) * 3 + 4 - 1) / 4);
 765   if (ret < 0) {
 766     lderr(cct) << "ceph_armor failed" << dendl;
 767     return ret;
 768   }
 769   tmp_dest[ret] = '\0';
 770   memcpy(dest, tmp_dest, size);
 771   dest[size-1] = '\0';
 772
 773   return 0;
 774 }
 775
 776 static const char alphanum_upper_table[]="0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ";
 777
 778 int gen_rand_alphanumeric_upper(CephContext *cct, char *dest, int size) /* size should be the required string size + 1 */
 779 {
 780   int ret = get_random_bytes(dest, size);
 781   if (ret < 0) {
 782     lderr(cct) << "cannot get random bytes: " << cpp_strerror(-ret) << dendl;
 783     return ret;
 784   }
 785
 786   int i;
 787   for (i=0; i<size - 1; i++) {
 788     int pos = (unsigned)dest[i];
 789     dest[i] = alphanum_upper_table[pos % (sizeof(alphanum_upper_table) - 1)];
 790   }
 791   dest[i] = '\0';
 792
 793   return 0;
 794 }
 795
 796 static const char alphanum_lower_table[]="0123456789abcdefghijklmnopqrstuvwxyz";
 797
 798 int gen_rand_alphanumeric_lower(CephContext *cct, char *dest, int size) /* size should be the required string size + 1 */
 799 {
 800   int ret = get_random_bytes(dest, size);
 801   if (ret < 0) {
 802     lderr(cct) << "cannot get random bytes: " << cpp_strerror(-ret) << dendl;
 803     return ret;
 804   }
 805
 806   int i;
 807   for (i=0; i<size - 1; i++) {
 808     int pos = (unsigned)dest[i];
 809     dest[i] = alphanum_lower_table[pos % (sizeof(alphanum_lower_table) - 1)];
 810   }
 811   dest[i] = '\0';
 812
 813   return 0;
 814 }
 815
 816 int gen_rand_alphanumeric_lower(CephContext *cct, string *str, int length)
 817 {
 818   char buf[length + 1];
 819   int ret = gen_rand_alphanumeric_lower(cct, buf, sizeof(buf));
 820   if (ret < 0) {
 821     return ret;
 822   }
 823   *str = buf;
 824   return 0;
 825 }
 826
 827 // this is basically a modified base64 charset, url friendly
 828 static const char alphanum_table[]="ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_";
 829
 830 int gen_rand_alphanumeric(CephContext *cct, char *dest, int size) /* size should be the required string size + 1 */
 831 {
 832   int ret = get_random_bytes(dest, size);
 833   if (ret < 0) {
 834     lderr(cct) << "cannot get random bytes: " << cpp_strerror(-ret) << dendl;
 835     return ret;
 836   }
 837
 838   int i;
 839   for (i=0; i<size - 1; i++) {
 840     int pos = (unsigned)dest[i];
 841     dest[i] = alphanum_table[pos & 63];
 842   }
 843   dest[i] = '\0';
 844
 845   return 0;
 846 }
 847
 848 static const char alphanum_no_underscore_table[]="ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-.";
 849
 850 int gen_rand_alphanumeric_no_underscore(CephContext *cct, char *dest, int size) /* size should be the required string size + 1 */
 851 {
 852   int ret = get_random_bytes(dest, size);
 853   if (ret < 0) {
 854     lderr(cct) << "cannot get random bytes: " << cpp_strerror(-ret) << dendl;
 855     return ret;
 856   }
 857
 858   int i;
 859   for (i=0; i<size - 1; i++) {
 860     int pos = (unsigned)dest[i];
 861     dest[i] = alphanum_no_underscore_table[pos & 63];
 862   }
 863   dest[i] = '\0';
 864
 865   return 0;
 866 }
 867
 868 static const char alphanum_plain_table[]="ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
 869
 870 int gen_rand_alphanumeric_plain(CephContext *cct, char *dest, int size) /* size should be the required string size + 1 */
 871 {
 872   int ret = get_random_bytes(dest, size);
 873   if (ret < 0) {
 874     lderr(cct) << "cannot get random bytes: " << cpp_strerror(-ret) << dendl;
 875     return ret;
 876   }
 877
 878   int i;
 879   for (i=0; i<size - 1; i++) {
 880     int pos = (unsigned)dest[i];
 881     dest[i] = alphanum_plain_table[pos % (sizeof(alphanum_plain_table) - 1)];
 882   }
 883   dest[i] = '\0';
 884
 885   return 0;
 886 }
 887
 888 int NameVal::parse()
 889 {
 890   auto delim_pos = str.find('=');
 891   int ret = 0;
 892
 893   if (delim_pos == string::npos) {
 894     name = str;
 895     val = "";
 896     ret = 1;
 897   } else {
 898     name = str.substr(0, delim_pos);
 899     val = str.substr(delim_pos + 1);
 900   }
 901
 902   return ret;
 903 }
 904
 905 int RGWHTTPArgs::parse()
 906 {
 907   int pos = 0;
 908   bool end = false;
 909
 910   if (str.empty())
 911     return 0;
 912
 913   if (str[pos] == '?')
 914     pos++;
 915
 916   while (!end) {
 917     int fpos = str.find('&', pos);
 918     if (fpos  < pos) {
 919        end = true;
 920        fpos = str.size();
 921     }
 922     std::string nameval = url_decode(str.substr(pos, fpos - pos), true);
 923     NameVal nv(std::move(nameval));
 924     int ret = nv.parse();
 925     if (ret >= 0) {
 926       string& name = nv.get_name();
 927       string& val = nv.get_val();
 928
 929       append(name, val);
 930     }
 931
 932     pos = fpos + 1;
 933   }
 934
 935   return 0;
 936 }
 937
 938 void RGWHTTPArgs::append(const string& name, const string& val)
 939 {
 940   if (name.compare(0, sizeof(RGW_SYS_PARAM_PREFIX) - 1, RGW_SYS_PARAM_PREFIX) == 0) {
 941     sys_val_map[name] = val;
 942   } else {
 943     val_map[name] = val;
 944   }
 945
 946   if ((name.compare("acl") == 0) ||
 947       (name.compare("cors") == 0) ||
 948       (name.compare("location") == 0) ||
 949       (name.compare("logging") == 0) ||
 950       (name.compare("usage") == 0) ||
 951       (name.compare("lifecycle") == 0) ||
 952       (name.compare("delete") == 0) ||
 953       (name.compare("uploads") == 0) ||
 954       (name.compare("partNumber") == 0) ||
 955       (name.compare("uploadId") == 0) ||
 956       (name.compare("versionId") == 0) ||
 957       (name.compare("start-date") == 0) ||
 958       (name.compare("end-date") == 0) ||
 959       (name.compare("versions") == 0) ||
 960       (name.compare("versioning") == 0) ||
 961       (name.compare("website") == 0) ||
 962       (name.compare("requestPayment") == 0) ||
 963       (name.compare("torrent") == 0) ||
 964       (name.compare("tagging") == 0)) {
 965     sub_resources[name] = val;
 966   } else if (name[0] == 'r') { // root of all evil
 967     if ((name.compare("response-content-type") == 0) ||
 968         (name.compare("response-content-language") == 0) ||
 969         (name.compare("response-expires") == 0) ||
 970         (name.compare("response-cache-control") == 0) ||
 971         (name.compare("response-content-disposition") == 0) ||
 972         (name.compare("response-content-encoding") == 0)) {
 973       sub_resources[name] = val;
 974       has_resp_modifier = true;
 975     }
 976   } else if  ((name.compare("subuser") == 0) ||
 977               (name.compare("key") == 0) ||
 978               (name.compare("caps") == 0) ||
 979               (name.compare("index") == 0) ||
 980               (name.compare("policy") == 0) ||
 981               (name.compare("quota") == 0) ||
 982               (name.compare("object") == 0)) {
 983
 984     if (!admin_subresource_added) {
 985       sub_resources[name] = "";
 986       admin_subresource_added = true;
 987     }
 988   }
 989 }
 990
 991 const string& RGWHTTPArgs::get(const string& name, bool *exists) const
 992 {
 993   auto iter = val_map.find(name);
 994   bool e = (iter != std::end(val_map));
 995   if (exists)
 996     *exists = e;
 997   if (e)
 998     return iter->second;
 999   return empty_str;
1000 }
1001
1002 boost::optional<const std::string&>
1003 RGWHTTPArgs::get_optional(const std::string& name) const
1004 {
1005   bool exists;
1006   const std::string& value = get(name, &exists);
1007   if (exists) {
1008     return value;
1009   } else {
1010     return boost::none;
1011   }
1012 }
1013
1014 int RGWHTTPArgs::get_bool(const string& name, bool *val, bool *exists)
1015 {
1016   map<string, string>::iterator iter;
1017   iter = val_map.find(name);
1018   bool e = (iter != val_map.end());
1019   if (exists)
1020     *exists = e;
1021
1022   if (e) {
1023     const char *s = iter->second.c_str();
1024
1025     if (strcasecmp(s, "false") == 0) {
1026       *val = false;
1027     } else if (strcasecmp(s, "true") == 0) {
1028       *val = true;
1029     } else {
1030       return -EINVAL;
1031     }
1032   }
1033
1034   return 0;
1035 }
1036
1037 int RGWHTTPArgs::get_bool(const char *name, bool *val, bool *exists)
1038 {
1039   string s(name);
1040   return get_bool(s, val, exists);
1041 }
1042
1043 void RGWHTTPArgs::get_bool(const char *name, bool *val, bool def_val)
1044 {
1045   bool exists = false;
1046   if ((get_bool(name, val, &exists) < 0) ||
1047       !exists) {
1048     *val = def_val;
1049   }
1050 }
1051
1052 string RGWHTTPArgs::sys_get(const string& name, bool * const exists) const
1053 {
1054   const auto iter = sys_val_map.find(name);
1055   const bool e = (iter != sys_val_map.end());
1056
1057   if (exists) {
1058     *exists = e;
1059   }
1060
1061   return e ? iter->second : string();
1062 }
1063
1064 bool verify_user_permission(struct req_state * const s,
1065                             RGWAccessControlPolicy * const user_acl,
1066                             const int perm)
1067 {
1068   /* S3 doesn't support account ACLs. */
1069   if (!user_acl)
1070     return true;
1071
1072   if ((perm & (int)s->perm_mask) != perm)
1073     return false;
1074
1075   return user_acl->verify_permission(*s->auth.identity, perm, perm);
1076 }
1077
1078 bool verify_user_permission(struct req_state * const s,
1079                             const int perm)
1080 {
1081   return verify_user_permission(s, s->user_acl.get(), perm);
1082 }
1083
1084 bool verify_requester_payer_permission(struct req_state *s)
1085 {
1086   if (!s->bucket_info.requester_pays)
1087     return true;
1088
1089   if (s->auth.identity->is_owner_of(s->bucket_info.owner))
1090     return true;
1091
1092   if (s->auth.identity->is_anonymous()) {
1093     return false;
1094   }
1095
1096   const char *request_payer = s->info.env->get("HTTP_X_AMZ_REQUEST_PAYER");
1097   if (!request_payer) {
1098     bool exists;
1099     request_payer = s->info.args.get("x-amz-request-payer", &exists).c_str();
1100     if (!exists) {
1101       return false;
1102     }
1103   }
1104
1105   if (strcasecmp(request_payer, "requester") == 0) {
1106     return true;
1107   }
1108
1109   return false;
1110 }
1111
1112 bool verify_bucket_permission(struct req_state * const s,
1113                               const rgw_bucket& bucket,
1114                               RGWAccessControlPolicy * const user_acl,
1115                               RGWAccessControlPolicy * const bucket_acl,
1116                               const optional<Policy>& bucket_policy,
1117                               const uint64_t op)
1118 {
1119   if (!verify_requester_payer_permission(s))
1120     return false;
1121
1122   if (bucket_policy) {
1123     auto r = bucket_policy->eval(s->env, *s->auth.identity, op, ARN(bucket));
1124     if (r == Effect::Allow)
1125       // It looks like S3 ACLs only GRANT permissions rather than
1126       // denying them, so this should be safe.
1127       return true;
1128     else if (r == Effect::Deny)
1129       return false;
1130   }
1131
1132   const auto perm = op_to_perm(op);
1133
1134   return verify_bucket_permission_no_policy(s, user_acl, bucket_acl, perm);
1135 }
1136
1137 bool verify_bucket_permission_no_policy(struct req_state * const s,
1138                                         RGWAccessControlPolicy * const user_acl,
1139                                         RGWAccessControlPolicy * const bucket_acl,
1140                                         const int perm)
1141 {
1142   if (!bucket_acl)
1143     return false;
1144
1145   if ((perm & (int)s->perm_mask) != perm)
1146     return false;
1147
1148   if (bucket_acl->verify_permission(*s->auth.identity, perm, perm,
1149                                     s->info.env->get("HTTP_REFERER")))
1150     return true;
1151
1152   if (!user_acl)
1153     return false;
1154
1155   return user_acl->verify_permission(*s->auth.identity, perm, perm);
1156 }
1157
1158 bool verify_bucket_permission_no_policy(struct req_state * const s, const int perm)
1159 {
1160   if (!verify_requester_payer_permission(s))
1161     return false;
1162
1163   return verify_bucket_permission_no_policy(s,
1164                                             s->user_acl.get(),
1165                                             s->bucket_acl.get(),
1166                                             perm);
1167 }
1168
1169 bool verify_bucket_permission(struct req_state * const s, const uint64_t op)
1170 {
1171   return verify_bucket_permission(s,
1172                                   s->bucket,
1173                                   s->user_acl.get(),
1174                                   s->bucket_acl.get(),
1175                                   s->iam_policy,
1176                                   op);
1177 }
1178
1179 static inline bool check_deferred_bucket_perms(struct req_state * const s,
1180                                                const rgw_bucket& bucket,
1181                                                RGWAccessControlPolicy * const user_acl,
1182                                                RGWAccessControlPolicy * const bucket_acl,
1183                                                const optional<Policy>& bucket_policy,
1184                                                const uint8_t deferred_check,
1185                                                const uint64_t op)
1186 {
1187   return (s->defer_to_bucket_acls == deferred_check \
1188           && verify_bucket_permission(s, bucket, user_acl, bucket_acl, bucket_policy, op));
1189 }
1190
1191 static inline bool check_deferred_bucket_only_acl(struct req_state * const s,
1192                                                   RGWAccessControlPolicy * const user_acl,
1193                                                   RGWAccessControlPolicy * const bucket_acl,
1194                                                   const uint8_t deferred_check,
1195                                                   const int perm)
1196 {
1197   return (s->defer_to_bucket_acls == deferred_check \
1198           && verify_bucket_permission_no_policy(s, user_acl, bucket_acl, perm));
1199 }
1200
1201 bool verify_object_permission(struct req_state * const s,
1202                               const rgw_obj& obj,
1203                               RGWAccessControlPolicy * const user_acl,
1204                               RGWAccessControlPolicy * const bucket_acl,
1205                               RGWAccessControlPolicy * const object_acl,
1206                               const optional<Policy>& bucket_policy,
1207                               const uint64_t op)
1208 {
1209   if (!verify_requester_payer_permission(s))
1210     return false;
1211
1212   if (bucket_policy) {
1213     auto r = bucket_policy->eval(s->env, *s->auth.identity, op, ARN(obj));
1214     if (r == Effect::Allow)
1215       // It looks like S3 ACLs only GRANT permissions rather than
1216       // denying them, so this should be safe.
1217       return true;
1218     else if (r == Effect::Deny)
1219       return false;
1220   }
1221
1222   const auto perm = op_to_perm(op);
1223
1224   if (check_deferred_bucket_perms(s, obj.bucket, user_acl, bucket_acl, bucket_policy,
1225                                   RGW_DEFER_TO_BUCKET_ACLS_RECURSE, op) ||
1226       check_deferred_bucket_perms(s, obj.bucket, user_acl, bucket_acl, bucket_policy,
1227                                   RGW_DEFER_TO_BUCKET_ACLS_FULL_CONTROL, rgw::IAM::s3All)) {
1228     return true;
1229   }
1230
1231   if (!object_acl) {
1232     return false;
1233   }
1234
1235   bool ret = object_acl->verify_permission(*s->auth.identity, s->perm_mask, perm);
1236   if (ret) {
1237     return true;
1238   }
1239
1240   if (!s->cct->_conf->rgw_enforce_swift_acls)
1241     return ret;
1242
1243   if ((perm & (int)s->perm_mask) != perm)
1244     return false;
1245
1246   int swift_perm = 0;
1247   if (perm & (RGW_PERM_READ | RGW_PERM_READ_ACP))
1248     swift_perm |= RGW_PERM_READ_OBJS;
1249   if (perm & RGW_PERM_WRITE)
1250     swift_perm |= RGW_PERM_WRITE_OBJS;
1251
1252   if (!swift_perm)
1253     return false;
1254
1255   /* we already verified the user mask above, so we pass swift_perm as the mask here,
1256      otherwise the mask might not cover the swift permissions bits */
1257   if (bucket_acl->verify_permission(*s->auth.identity, swift_perm, swift_perm,
1258                                     s->info.env->get("HTTP_REFERER")))
1259     return true;
1260
1261   if (!user_acl)
1262     return false;
1263
1264   return user_acl->verify_permission(*s->auth.identity, swift_perm, swift_perm);
1265 }
1266
1267 bool verify_object_permission_no_policy(struct req_state * const s,
1268                                         RGWAccessControlPolicy * const user_acl,
1269                                         RGWAccessControlPolicy * const bucket_acl,
1270                                         RGWAccessControlPolicy * const object_acl,
1271                                         const int perm)
1272 {
1273   if (check_deferred_bucket_only_acl(s, user_acl, bucket_acl, RGW_DEFER_TO_BUCKET_ACLS_RECURSE, perm) ||
1274       check_deferred_bucket_only_acl(s, user_acl, bucket_acl, RGW_DEFER_TO_BUCKET_ACLS_FULL_CONTROL, RGW_PERM_FULL_CONTROL)) {
1275     return true;
1276   }
1277
1278   if (!object_acl) {
1279     return false;
1280   }
1281
1282   bool ret = object_acl->verify_permission(*s->auth.identity, s->perm_mask, perm);
1283   if (ret) {
1284     return true;
1285   }
1286
1287   if (!s->cct->_conf->rgw_enforce_swift_acls)
1288     return ret;
1289
1290   if ((perm & (int)s->perm_mask) != perm)
1291     return false;
1292
1293   int swift_perm = 0;
1294   if (perm & (RGW_PERM_READ | RGW_PERM_READ_ACP))
1295     swift_perm |= RGW_PERM_READ_OBJS;
1296   if (perm & RGW_PERM_WRITE)
1297     swift_perm |= RGW_PERM_WRITE_OBJS;
1298
1299   if (!swift_perm)
1300     return false;
1301
1302   /* we already verified the user mask above, so we pass swift_perm as the mask here,
1303      otherwise the mask might not cover the swift permissions bits */
1304   if (bucket_acl->verify_permission(*s->auth.identity, swift_perm, swift_perm,
1305                                     s->info.env->get("HTTP_REFERER")))
1306     return true;
1307
1308   if (!user_acl)
1309     return false;
1310
1311   return user_acl->verify_permission(*s->auth.identity, swift_perm, swift_perm);
1312 }
1313
1314 bool verify_object_permission_no_policy(struct req_state *s, int perm)
1315 {
1316   if (!verify_requester_payer_permission(s))
1317     return false;
1318
1319   return verify_object_permission_no_policy(s,
1320                                             s->user_acl.get(),
1321                                             s->bucket_acl.get(),
1322                                             s->object_acl.get(),
1323                                             perm);
1324 }
1325
1326 bool verify_object_permission(struct req_state *s, uint64_t op)
1327 {
1328   return verify_object_permission(s,
1329                                   rgw_obj(s->bucket, s->object),
1330                                   s->user_acl.get(),
1331                                   s->bucket_acl.get(),
1332                                   s->object_acl.get(),
1333                                   s->iam_policy,
1334                                   op);
1335 }
1336
1337 class HexTable
1338 {
1339   char table[256];
1340
1341 public:
1342   HexTable() {
1343     memset(table, -1, sizeof(table));
1344     int i;
1345     for (i = '0'; i<='9'; i++)
1346       table[i] = i - '0';
1347     for (i = 'A'; i<='F'; i++)
1348       table[i] = i - 'A' + 0xa;
1349     for (i = 'a'; i<='f'; i++)
1350       table[i] = i - 'a' + 0xa;
1351   }
1352
1353   char to_num(char c) {
1354     return table[(int)c];
1355   }
1356 };
1357
1358 static char hex_to_num(char c)
1359 {
1360   static HexTable hex_table;
1361   return hex_table.to_num(c);
1362 }
1363
1364 std::string url_decode(const boost::string_view& src_str, bool in_query)
1365 {
1366   std::string dest_str;
1367   dest_str.reserve(src_str.length() + 1);
1368
1369   for (auto src = std::begin(src_str); src != std::end(src_str); ++src) {
1370     if (*src != '%') {
1371       if (!in_query || *src != '+') {
1372         if (*src == '?') {
1373           in_query = true;
1374         }
1375         dest_str.push_back(*src);
1376       } else {
1377         dest_str.push_back(' ');
1378       }
1379     } else {
1380       /* 3 == strlen("%%XX") */
1381       if (std::distance(src, std::end(src_str)) < 3) {
1382         break;
1383       }
1384
1385       src++;
1386       const char c1 = hex_to_num(*src++);
1387       const char c2 = hex_to_num(*src);
1388       if (c1 < 0 || c2 < 0) {
1389         return std::string();
1390       } else {
1391         dest_str.push_back(c1 << 4 | c2);
1392       }
1393     }
1394   }
1395
1396   return dest_str;
1397 }
1398
1399 void rgw_uri_escape_char(char c, string& dst)
1400 {
1401   char buf[16];
1402   snprintf(buf, sizeof(buf), "%%%.2X", (int)(unsigned char)c);
1403   dst.append(buf);
1404 }
1405
1406 static bool char_needs_url_encoding(char c)
1407 {
1408   if (c <= 0x20 || c >= 0x7f)
1409     return true;
1410
1411   switch (c) {
1412     case 0x22:
1413     case 0x23:
1414     case 0x25:
1415     case 0x26:
1416     case 0x2B:
1417     case 0x2C:
1418     case 0x2F:
1419     case 0x3A:
1420     case 0x3B:
1421     case 0x3C:
1422     case 0x3E:
1423     case 0x3D:
1424     case 0x3F:
1425     case 0x40:
1426     case 0x5B:
1427     case 0x5D:
1428     case 0x5C:
1429     case 0x5E:
1430     case 0x60:
1431     case 0x7B:
1432     case 0x7D:
1433       return true;
1434   }
1435   return false;
1436 }
1437
1438 void url_encode(const string& src, string& dst)
1439 {
1440   const char *p = src.c_str();
1441   for (unsigned i = 0; i < src.size(); i++, p++) {
1442     if (char_needs_url_encoding(*p)) {
1443       rgw_uri_escape_char(*p, dst);
1444       continue;
1445     }
1446
1447     dst.append(p, 1);
1448   }
1449 }
1450
1451 std::string url_encode(const std::string& src)
1452 {
1453   std::string dst;
1454   url_encode(src, dst);
1455
1456   return dst;
1457 }
1458
1459 string rgw_trim_whitespace(const string& src)
1460 {
1461   if (src.empty()) {
1462     return string();
1463   }
1464
1465   int start = 0;
1466   for (; start != (int)src.size(); start++) {
1467     if (!isspace(src[start]))
1468       break;
1469   }
1470
1471   int end = src.size() - 1;
1472   if (end < start) {
1473     return string();
1474   }
1475
1476   for (; end > start; end--) {
1477     if (!isspace(src[end]))
1478       break;
1479   }
1480
1481   return src.substr(start, end - start + 1);
1482 }
1483
1484 boost::string_view rgw_trim_whitespace(const boost::string_view& src)
1485 {
1486   boost::string_view res = src;
1487
1488   while (res.size() > 0 && std::isspace(res.front())) {
1489     res.remove_prefix(1);
1490   }
1491   while (res.size() > 0 && std::isspace(res.back())) {
1492     res.remove_suffix(1);
1493   }
1494   return res;
1495 }
1496
1497 string rgw_trim_quotes(const string& val)
1498 {
1499   string s = rgw_trim_whitespace(val);
1500   if (s.size() < 2)
1501     return s;
1502
1503   int start = 0;
1504   int end = s.size() - 1;
1505   int quotes_count = 0;
1506
1507   if (s[start] == '"') {
1508     start++;
1509     quotes_count++;
1510   }
1511   if (s[end] == '"') {
1512     end--;
1513     quotes_count++;
1514   }
1515   if (quotes_count == 2) {
1516     return s.substr(start, end - start + 1);
1517   }
1518   return s;
1519 }
1520
1521 struct rgw_name_to_flag {
1522   const char *type_name;
1523   uint32_t flag;
1524 };
1525
1526 static int parse_list_of_flags(struct rgw_name_to_flag *mapping,
1527                                const string& str, uint32_t *perm)
1528 {
1529   list<string> strs;
1530   get_str_list(str, strs);
1531   list<string>::iterator iter;
1532   uint32_t v = 0;
1533   for (iter = strs.begin(); iter != strs.end(); ++iter) {
1534     string& s = *iter;
1535     for (int i = 0; mapping[i].type_name; i++) {
1536       if (s.compare(mapping[i].type_name) == 0)
1537         v |= mapping[i].flag;
1538     }
1539   }
1540
1541   *perm = v;
1542   return 0;
1543 }
1544
1545 static struct rgw_name_to_flag cap_names[] = { {"*",     RGW_CAP_ALL},
1546                   {"read",  RGW_CAP_READ},
1547                   {"write", RGW_CAP_WRITE},
1548                   {NULL, 0} };
1549
1550 int RGWUserCaps::parse_cap_perm(const string& str, uint32_t *perm)
1551 {
1552   return parse_list_of_flags(cap_names, str, perm);
1553 }
1554
1555 int RGWUserCaps::get_cap(const string& cap, string& type, uint32_t *pperm)
1556 {
1557   int pos = cap.find('=');
1558   if (pos >= 0) {
1559     type = rgw_trim_whitespace(cap.substr(0, pos));
1560   }
1561
1562   if (!is_valid_cap_type(type))
1563     return -ERR_INVALID_CAP;
1564
1565   string cap_perm;
1566   uint32_t perm = 0;
1567   if (pos < (int)cap.size() - 1) {
1568     cap_perm = cap.substr(pos + 1);
1569     int r = RGWUserCaps::parse_cap_perm(cap_perm, &perm);
1570     if (r < 0)
1571       return r;
1572   }
1573
1574   *pperm = perm;
1575
1576   return 0;
1577 }
1578
1579 int RGWUserCaps::add_cap(const string& cap)
1580 {
1581   uint32_t perm;
1582   string type;
1583
1584   int r = get_cap(cap, type, &perm);
1585   if (r < 0)
1586     return r;
1587
1588   caps[type] |= perm;
1589
1590   return 0;
1591 }
1592
1593 int RGWUserCaps::remove_cap(const string& cap)
1594 {
1595   uint32_t perm;
1596   string type;
1597
1598   int r = get_cap(cap, type, &perm);
1599   if (r < 0)
1600     return r;
1601
1602   map<string, uint32_t>::iterator iter = caps.find(type);
1603   if (iter == caps.end())
1604     return 0;
1605
1606   uint32_t& old_perm = iter->second;
1607   old_perm &= ~perm;
1608   if (!old_perm)
1609     caps.erase(iter);
1610
1611   return 0;
1612 }
1613
1614 int RGWUserCaps::add_from_string(const string& str)
1615 {
1616   int start = 0;
1617   do {
1618     auto end = str.find(';', start);
1619     if (end == string::npos)
1620       end = str.size();
1621
1622     int r = add_cap(str.substr(start, end - start));
1623     if (r < 0)
1624       return r;
1625
1626     start = end + 1;
1627   } while (start < (int)str.size());
1628
1629   return 0;
1630 }
1631
1632 int RGWUserCaps::remove_from_string(const string& str)
1633 {
1634   int start = 0;
1635   do {
1636     auto end = str.find(';', start);
1637     if (end == string::npos)
1638       end = str.size();
1639
1640     int r = remove_cap(str.substr(start, end - start));
1641     if (r < 0)
1642       return r;
1643
1644     start = end + 1;
1645   } while (start < (int)str.size());
1646
1647   return 0;
1648 }
1649
1650 void RGWUserCaps::dump(Formatter *f) const
1651 {
1652   dump(f, "caps");
1653 }
1654
1655 void RGWUserCaps::dump(Formatter *f, const char *name) const
1656 {
1657   f->open_array_section(name);
1658   map<string, uint32_t>::const_iterator iter;
1659   for (iter = caps.begin(); iter != caps.end(); ++iter)
1660   {
1661     f->open_object_section("cap");
1662     f->dump_string("type", iter->first);
1663     uint32_t perm = iter->second;
1664     string perm_str;
1665     for (int i=0; cap_names[i].type_name; i++) {
1666       if ((perm & cap_names[i].flag) == cap_names[i].flag) {
1667         if (perm_str.size())
1668           perm_str.append(", ");
1669
1670         perm_str.append(cap_names[i].type_name);
1671         perm &= ~cap_names[i].flag;
1672       }
1673     }
1674     if (perm_str.empty())
1675       perm_str = "<none>";
1676
1677     f->dump_string("perm", perm_str);
1678     f->close_section();
1679   }
1680
1681   f->close_section();
1682 }
1683
1684 struct RGWUserCap {
1685   string type;
1686   uint32_t perm;
1687
1688   void decode_json(JSONObj *obj) {
1689     JSONDecoder::decode_json("type", type, obj);
1690     string perm_str;
1691     JSONDecoder::decode_json("perm", perm_str, obj);
1692     if (RGWUserCaps::parse_cap_perm(perm_str, &perm) < 0) {
1693       throw JSONDecoder::err("failed to parse permissions");
1694     }
1695   }
1696 };
1697
1698 void RGWUserCaps::decode_json(JSONObj *obj)
1699 {
1700   list<RGWUserCap> caps_list;
1701   decode_json_obj(caps_list, obj);
1702
1703   list<RGWUserCap>::iterator iter;
1704   for (iter = caps_list.begin(); iter != caps_list.end(); ++iter) {
1705     RGWUserCap& cap = *iter;
1706     caps[cap.type] = cap.perm;
1707   }
1708 }
1709
1710 int RGWUserCaps::check_cap(const string& cap, uint32_t perm)
1711 {
1712   map<string, uint32_t>::iterator iter = caps.find(cap);
1713
1714   if ((iter == caps.end()) ||
1715       (iter->second & perm) != perm) {
1716     return -EPERM;
1717   }
1718
1719   return 0;
1720 }
1721
1722 bool RGWUserCaps::is_valid_cap_type(const string& tp)
1723 {
1724   static const char *cap_type[] = { "user",
1725                                     "users",
1726                                     "buckets",
1727                                     "metadata",
1728                                     "usage",
1729                                     "zone",
1730                                     "bilog",
1731                                     "mdlog",
1732                                     "datalog",
1733                                     "opstate" };
1734
1735   for (unsigned int i = 0; i < sizeof(cap_type) / sizeof(char *); ++i) {
1736     if (tp.compare(cap_type[i]) == 0) {
1737       return true;
1738     }
1739   }
1740
1741   return false;
1742 }
1743
1744 static ssize_t unescape_str(const string& s, ssize_t ofs, char esc_char, char special_char, string *dest)
1745 {
1746   const char *src = s.c_str();
1747   char dest_buf[s.size() + 1];
1748   char *destp = dest_buf;
1749   bool esc = false;
1750
1751   dest_buf[0] = '\0';
1752
1753   for (size_t i = ofs; i < s.size(); i++) {
1754     char c = src[i];
1755     if (!esc && c == esc_char) {
1756       esc = true;
1757       continue;
1758     }
1759     if (!esc && c == special_char) {
1760       *destp = '\0';
1761       *dest = dest_buf;
1762       return (ssize_t)i + 1;
1763     }
1764     *destp++ = c;
1765     esc = false;
1766   }
1767   *destp = '\0';
1768   *dest = dest_buf;
1769   return string::npos;
1770 }
1771
1772 static void escape_str(const string& s, char esc_char, char special_char, string *dest)
1773 {
1774   const char *src = s.c_str();
1775   char dest_buf[s.size() * 2 + 1];
1776   char *destp = dest_buf;
1777
1778   for (size_t i = 0; i < s.size(); i++) {
1779     char c = src[i];
1780     if (c == esc_char || c == special_char) {
1781       *destp++ = esc_char;
1782     }
1783     *destp++ = c;
1784   }
1785   *destp++ = '\0';
1786   *dest = dest_buf;
1787 }
1788
1789 void rgw_pool::from_str(const string& s)
1790 {
1791   size_t pos = unescape_str(s, 0, '\\', ':', &name);
1792   if (pos != string::npos) {
1793     pos = unescape_str(s, pos, '\\', ':', &ns);
1794     /* ignore return; if pos != string::npos it means that we had a colon
1795      * in the middle of ns that wasn't escaped, we're going to stop there
1796      */
1797   }
1798 }
1799
1800 string rgw_pool::to_str() const
1801 {
1802   string esc_name;
1803   escape_str(name, '\\', ':', &esc_name);
1804   if (ns.empty()) {
1805     return esc_name;
1806   }
1807   string esc_ns;
1808   escape_str(ns, '\\', ':', &esc_ns);
1809   return esc_name + ":" + esc_ns;
1810 }
1811
1812 void rgw_raw_obj::decode_from_rgw_obj(bufferlist::iterator& bl)
1813 {
1814   rgw_obj old_obj;
1815   ::decode(old_obj, bl);
1816
1817   get_obj_bucket_and_oid_loc(old_obj, oid, loc);
1818   pool = old_obj.get_explicit_data_pool();
1819 }
1820
1821 std::string rgw_bucket::get_key(char tenant_delim, char id_delim) const
1822 {
1823   static constexpr size_t shard_len{12}; // ":4294967295\0"
1824   const size_t max_len = tenant.size() + sizeof(tenant_delim) +
1825       name.size() + sizeof(id_delim) + bucket_id.size() + shard_len;
1826
1827   std::string key;
1828   key.reserve(max_len);
1829   if (!tenant.empty() && tenant_delim) {
1830     key.append(tenant);
1831     key.append(1, tenant_delim);
1832   }
1833   key.append(name);
1834   if (!bucket_id.empty() && id_delim) {
1835     key.append(1, id_delim);
1836     key.append(bucket_id);
1837   }
1838   return key;
1839 }
1840
1841 std::string rgw_bucket_shard::get_key(char tenant_delim, char id_delim,
1842                                       char shard_delim) const
1843 {
1844   auto key = bucket.get_key(tenant_delim, id_delim);
1845   if (shard_id >= 0 && shard_delim) {
1846     key.append(1, shard_delim);
1847     key.append(std::to_string(shard_id));
1848   }
1849   return key;
1850 }
1851
1852 static struct rgw_name_to_flag op_type_mapping[] = { {"*",  RGW_OP_TYPE_ALL},
1853                   {"read",  RGW_OP_TYPE_READ},
1854                   {"write", RGW_OP_TYPE_WRITE},
1855                   {"delete", RGW_OP_TYPE_DELETE},
1856                   {NULL, 0} };
1857
1858
1859 int rgw_parse_op_type_list(const string& str, uint32_t *perm)
1860 {
1861   return parse_list_of_flags(op_type_mapping, str, perm);
1862 }
1863
1864 bool match_policy(boost::string_view pattern, boost::string_view input,
1865                   uint32_t flag)
1866 {
1867   const uint32_t flag2 = flag & (MATCH_POLICY_ACTION|MATCH_POLICY_ARN) ?
1868       MATCH_CASE_INSENSITIVE : 0;
1869
1870   const auto npos = boost::string_view::npos;
1871   boost::string_view::size_type last_pos_input = 0, last_pos_pattern = 0;
1872   while (true) {
1873     auto cur_pos_input = input.find(":", last_pos_input);
1874     auto cur_pos_pattern = pattern.find(":", last_pos_pattern);
1875
1876     auto substr_input = input.substr(last_pos_input, cur_pos_input);
1877     auto substr_pattern = pattern.substr(last_pos_pattern, cur_pos_pattern);
1878
1879     if (!match_wildcards(substr_pattern, substr_input, flag2))
1880       return false;
1881
1882     if (cur_pos_pattern == npos)
1883       return cur_pos_input == npos;
1884     if (cur_pos_input == npos)
1885       return false;
1886
1887     last_pos_pattern = cur_pos_pattern + 1;
1888     last_pos_input = cur_pos_input + 1;
1889   }
1890 }