ceph/src/rgw/rgw_common.h

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab ft=cpp
   3
   4 /*
   5  * Ceph - scalable distributed file system
   6  *
   7  * Copyright (C) 2004-2009 Sage Weil <sage@newdream.net>
   8  * Copyright (C) 2015 Yehuda Sadeh <yehuda@redhat.com>
   9  *
  10  * This is free software; you can redistribute it and/or
  11  * modify it under the terms of the GNU Lesser General Public
  12  * License version 2.1, as published by the Free Software
  13  * Foundation.  See file COPYING.
  14  *
  15  */
  16
  17 #ifndef CEPH_RGW_COMMON_H
  18 #define CEPH_RGW_COMMON_H
  19
  20 #include <array>
  21 #include <string_view>
  22
  23 #include "common/ceph_crypto.h"
  24 #include "common/random_string.h"
  25 #include "rgw_acl.h"
  26 #include "rgw_bucket_layout.h"
  27 #include "rgw_cors.h"
  28 #include "rgw_iam_policy.h"
  29 #include "rgw_quota.h"
  30 #include "rgw_string.h"
  31 #include "common/async/yield_context.h"
  32 #include "rgw_website.h"
  33 #include "rgw_object_lock.h"
  34 #include "rgw_tag.h"
  35 #include "rgw_op_type.h"
  36 #include "rgw_sync_policy.h"
  37 #include "cls/version/cls_version_types.h"
  38 #include "cls/user/cls_user_types.h"
  39 #include "cls/rgw/cls_rgw_types.h"
  40 #include "include/rados/librados.hpp"
  41 #include "rgw_public_access.h"
  42
  43 namespace ceph {
  44   class Formatter;
  45 }
  46
  47 namespace rgw::sal {
  48   class RGWUser;
  49   class RGWBucket;
  50   class RGWObject;
  51   using RGWAttrs = std::map<std::string, ceph::buffer::list>;
  52 }
  53
  54 using ceph::crypto::MD5;
  55
  56 #define RGW_ATTR_PREFIX  "user.rgw."
  57
  58 #define RGW_HTTP_RGWX_ATTR_PREFIX "RGWX_ATTR_"
  59 #define RGW_HTTP_RGWX_ATTR_PREFIX_OUT "Rgwx-Attr-"
  60
  61 #define RGW_AMZ_PREFIX "x-amz-"
  62 #define RGW_AMZ_META_PREFIX RGW_AMZ_PREFIX "meta-"
  63 #define RGW_AMZ_WEBSITE_REDIRECT_LOCATION RGW_AMZ_PREFIX "website-redirect-location"
  64 #define RGW_AMZ_TAG_COUNT RGW_AMZ_PREFIX "tagging-count"
  65
  66 #define RGW_SYS_PARAM_PREFIX "rgwx-"
  67
  68 #define RGW_ATTR_ACL            RGW_ATTR_PREFIX "acl"
  69 #define RGW_ATTR_LC            RGW_ATTR_PREFIX "lc"
  70 #define RGW_ATTR_CORS           RGW_ATTR_PREFIX "cors"
  71 #define RGW_ATTR_ETAG           RGW_ATTR_PREFIX "etag"
  72 #define RGW_ATTR_BUCKETS        RGW_ATTR_PREFIX "buckets"
  73 #define RGW_ATTR_META_PREFIX    RGW_ATTR_PREFIX RGW_AMZ_META_PREFIX
  74 #define RGW_ATTR_CONTENT_TYPE   RGW_ATTR_PREFIX "content_type"
  75 #define RGW_ATTR_CACHE_CONTROL  RGW_ATTR_PREFIX "cache_control"
  76 #define RGW_ATTR_CONTENT_DISP   RGW_ATTR_PREFIX "content_disposition"
  77 #define RGW_ATTR_CONTENT_ENC    RGW_ATTR_PREFIX "content_encoding"
  78 #define RGW_ATTR_CONTENT_LANG   RGW_ATTR_PREFIX "content_language"
  79 #define RGW_ATTR_EXPIRES        RGW_ATTR_PREFIX "expires"
  80 #define RGW_ATTR_DELETE_AT      RGW_ATTR_PREFIX "delete_at"
  81 #define RGW_ATTR_ID_TAG         RGW_ATTR_PREFIX "idtag"
  82 #define RGW_ATTR_TAIL_TAG       RGW_ATTR_PREFIX "tail_tag"
  83 #define RGW_ATTR_SHADOW_OBJ     RGW_ATTR_PREFIX "shadow_name"
  84 #define RGW_ATTR_MANIFEST       RGW_ATTR_PREFIX "manifest"
  85 #define RGW_ATTR_USER_MANIFEST  RGW_ATTR_PREFIX "user_manifest"
  86 #define RGW_ATTR_AMZ_WEBSITE_REDIRECT_LOCATION  RGW_ATTR_PREFIX RGW_AMZ_WEBSITE_REDIRECT_LOCATION
  87 #define RGW_ATTR_SLO_MANIFEST   RGW_ATTR_PREFIX "slo_manifest"
  88 /* Information whether an object is SLO or not must be exposed to
  89  * user through custom HTTP header named X-Static-Large-Object. */
  90 #define RGW_ATTR_SLO_UINDICATOR RGW_ATTR_META_PREFIX "static-large-object"
  91 #define RGW_ATTR_X_ROBOTS_TAG   RGW_ATTR_PREFIX "x-robots-tag"
  92 #define RGW_ATTR_STORAGE_CLASS  RGW_ATTR_PREFIX "storage_class"
  93
  94 /* S3 Object Lock*/
  95 #define RGW_ATTR_OBJECT_LOCK        RGW_ATTR_PREFIX "object-lock"
  96 #define RGW_ATTR_OBJECT_RETENTION   RGW_ATTR_PREFIX "object-retention"
  97 #define RGW_ATTR_OBJECT_LEGAL_HOLD  RGW_ATTR_PREFIX "object-legal-hold"
  98
  99
 100 #define RGW_ATTR_PG_VER         RGW_ATTR_PREFIX "pg_ver"
 101 #define RGW_ATTR_SOURCE_ZONE    RGW_ATTR_PREFIX "source_zone"
 102 #define RGW_ATTR_TAGS           RGW_ATTR_PREFIX RGW_AMZ_PREFIX "tagging"
 103
 104 #define RGW_ATTR_TEMPURL_KEY1   RGW_ATTR_META_PREFIX "temp-url-key"
 105 #define RGW_ATTR_TEMPURL_KEY2   RGW_ATTR_META_PREFIX "temp-url-key-2"
 106
 107 /* Account/container quota of the Swift API. */
 108 #define RGW_ATTR_QUOTA_NOBJS    RGW_ATTR_META_PREFIX "quota-count"
 109 #define RGW_ATTR_QUOTA_MSIZE    RGW_ATTR_META_PREFIX "quota-bytes"
 110
 111 /* Static Web Site of Swift API. */
 112 #define RGW_ATTR_WEB_INDEX      RGW_ATTR_META_PREFIX "web-index"
 113 #define RGW_ATTR_WEB_ERROR      RGW_ATTR_META_PREFIX "web-error"
 114 #define RGW_ATTR_WEB_LISTINGS   RGW_ATTR_META_PREFIX "web-listings"
 115 #define RGW_ATTR_WEB_LIST_CSS   RGW_ATTR_META_PREFIX "web-listings-css"
 116 #define RGW_ATTR_SUBDIR_MARKER  RGW_ATTR_META_PREFIX "web-directory-type"
 117
 118 #define RGW_ATTR_OLH_PREFIX     RGW_ATTR_PREFIX "olh."
 119
 120 #define RGW_ATTR_OLH_INFO       RGW_ATTR_OLH_PREFIX "info"
 121 #define RGW_ATTR_OLH_VER        RGW_ATTR_OLH_PREFIX "ver"
 122 #define RGW_ATTR_OLH_ID_TAG     RGW_ATTR_OLH_PREFIX "idtag"
 123 #define RGW_ATTR_OLH_PENDING_PREFIX RGW_ATTR_OLH_PREFIX "pending."
 124
 125 #define RGW_ATTR_COMPRESSION    RGW_ATTR_PREFIX "compression"
 126
 127 #define RGW_ATTR_APPEND_PART_NUM    RGW_ATTR_PREFIX "append_part_num"
 128
 129 /* IAM Policy */
 130 #define RGW_ATTR_IAM_POLICY     RGW_ATTR_PREFIX "iam-policy"
 131 #define RGW_ATTR_USER_POLICY    RGW_ATTR_PREFIX "user-policy"
 132 #define RGW_ATTR_PUBLIC_ACCESS  RGW_ATTR_PREFIX "public-access"
 133
 134 /* RGW File Attributes */
 135 #define RGW_ATTR_UNIX_KEY1      RGW_ATTR_PREFIX "unix-key1"
 136 #define RGW_ATTR_UNIX1          RGW_ATTR_PREFIX "unix1"
 137
 138 #define RGW_ATTR_CRYPT_PREFIX   RGW_ATTR_PREFIX "crypt."
 139 #define RGW_ATTR_CRYPT_MODE     RGW_ATTR_CRYPT_PREFIX "mode"
 140 #define RGW_ATTR_CRYPT_KEYMD5   RGW_ATTR_CRYPT_PREFIX "keymd5"
 141 #define RGW_ATTR_CRYPT_KEYID    RGW_ATTR_CRYPT_PREFIX "keyid"
 142 #define RGW_ATTR_CRYPT_KEYSEL   RGW_ATTR_CRYPT_PREFIX "keysel"
 143 #define RGW_ATTR_CRYPT_CONTEXT  RGW_ATTR_CRYPT_PREFIX "context"
 144 #define RGW_ATTR_CRYPT_DATAKEY  RGW_ATTR_CRYPT_PREFIX "datakey"
 145
 146
 147 #define RGW_FORMAT_PLAIN        0
 148 #define RGW_FORMAT_XML          1
 149 #define RGW_FORMAT_JSON         2
 150 #define RGW_FORMAT_HTML         3
 151
 152 #define RGW_CAP_READ            0x1
 153 #define RGW_CAP_WRITE           0x2
 154 #define RGW_CAP_ALL             (RGW_CAP_READ | RGW_CAP_WRITE)
 155
 156 #define RGW_REST_SWIFT          0x1
 157 #define RGW_REST_SWIFT_AUTH     0x2
 158 #define RGW_REST_S3             0x4
 159 #define RGW_REST_WEBSITE     0x8
 160 #define RGW_REST_STS            0x10
 161 #define RGW_REST_IAM            0x20
 162
 163 #define RGW_SUSPENDED_USER_AUID (uint64_t)-2
 164
 165 #define RGW_OP_TYPE_READ         0x01
 166 #define RGW_OP_TYPE_WRITE        0x02
 167 #define RGW_OP_TYPE_DELETE       0x04
 168
 169 #define RGW_OP_TYPE_MODIFY       (RGW_OP_TYPE_WRITE | RGW_OP_TYPE_DELETE)
 170 #define RGW_OP_TYPE_ALL          (RGW_OP_TYPE_READ | RGW_OP_TYPE_WRITE | RGW_OP_TYPE_DELETE)
 171
 172 #define RGW_DEFAULT_MAX_BUCKETS 1000
 173
 174 #define RGW_DEFER_TO_BUCKET_ACLS_RECURSE 1
 175 #define RGW_DEFER_TO_BUCKET_ACLS_FULL_CONTROL 2
 176
 177 #define STATUS_CREATED           1900
 178 #define STATUS_ACCEPTED          1901
 179 #define STATUS_NO_CONTENT        1902
 180 #define STATUS_PARTIAL_CONTENT   1903
 181 #define STATUS_REDIRECT          1904
 182 #define STATUS_NO_APPLY          1905
 183 #define STATUS_APPLIED           1906
 184
 185 #define ERR_INVALID_BUCKET_NAME  2000
 186 #define ERR_INVALID_OBJECT_NAME  2001
 187 #define ERR_NO_SUCH_BUCKET       2002
 188 #define ERR_METHOD_NOT_ALLOWED   2003
 189 #define ERR_INVALID_DIGEST       2004
 190 #define ERR_BAD_DIGEST           2005
 191 #define ERR_UNRESOLVABLE_EMAIL   2006
 192 #define ERR_INVALID_PART         2007
 193 #define ERR_INVALID_PART_ORDER   2008
 194 #define ERR_NO_SUCH_UPLOAD       2009
 195 #define ERR_REQUEST_TIMEOUT      2010
 196 #define ERR_LENGTH_REQUIRED      2011
 197 #define ERR_REQUEST_TIME_SKEWED  2012
 198 #define ERR_BUCKET_EXISTS        2013
 199 #define ERR_BAD_URL              2014
 200 #define ERR_PRECONDITION_FAILED  2015
 201 #define ERR_NOT_MODIFIED         2016
 202 #define ERR_INVALID_UTF8         2017
 203 #define ERR_UNPROCESSABLE_ENTITY 2018
 204 #define ERR_TOO_LARGE            2019
 205 #define ERR_TOO_MANY_BUCKETS     2020
 206 #define ERR_INVALID_REQUEST      2021
 207 #define ERR_TOO_SMALL            2022
 208 #define ERR_NOT_FOUND            2023
 209 #define ERR_PERMANENT_REDIRECT   2024
 210 #define ERR_LOCKED               2025
 211 #define ERR_QUOTA_EXCEEDED       2026
 212 #define ERR_SIGNATURE_NO_MATCH   2027
 213 #define ERR_INVALID_ACCESS_KEY   2028
 214 #define ERR_MALFORMED_XML        2029
 215 #define ERR_USER_EXIST           2030
 216 #define ERR_NOT_SLO_MANIFEST     2031
 217 #define ERR_EMAIL_EXIST          2032
 218 #define ERR_KEY_EXIST            2033
 219 #define ERR_INVALID_SECRET_KEY   2034
 220 #define ERR_INVALID_KEY_TYPE     2035
 221 #define ERR_INVALID_CAP          2036
 222 #define ERR_INVALID_TENANT_NAME  2037
 223 #define ERR_WEBSITE_REDIRECT     2038
 224 #define ERR_NO_SUCH_WEBSITE_CONFIGURATION 2039
 225 #define ERR_AMZ_CONTENT_SHA256_MISMATCH 2040
 226 #define ERR_NO_SUCH_LC           2041
 227 #define ERR_NO_SUCH_USER         2042
 228 #define ERR_NO_SUCH_SUBUSER      2043
 229 #define ERR_MFA_REQUIRED         2044
 230 #define ERR_NO_SUCH_CORS_CONFIGURATION 2045
 231 #define ERR_NO_SUCH_OBJECT_LOCK_CONFIGURATION  2046
 232 #define ERR_INVALID_RETENTION_PERIOD 2047
 233 #define ERR_USER_SUSPENDED       2100
 234 #define ERR_INTERNAL_ERROR       2200
 235 #define ERR_NOT_IMPLEMENTED      2201
 236 #define ERR_SERVICE_UNAVAILABLE  2202
 237 #define ERR_ROLE_EXISTS          2203
 238 #define ERR_MALFORMED_DOC        2204
 239 #define ERR_NO_ROLE_FOUND        2205
 240 #define ERR_DELETE_CONFLICT      2206
 241 #define ERR_NO_SUCH_BUCKET_POLICY  2207
 242 #define ERR_INVALID_LOCATION_CONSTRAINT 2208
 243 #define ERR_TAG_CONFLICT         2209
 244 #define ERR_INVALID_TAG          2210
 245 #define ERR_ZERO_IN_URL          2211
 246 #define ERR_MALFORMED_ACL_ERROR  2212
 247 #define ERR_ZONEGROUP_DEFAULT_PLACEMENT_MISCONFIGURATION 2213
 248 #define ERR_INVALID_ENCRYPTION_ALGORITHM                 2214
 249 #define ERR_INVALID_CORS_RULES_ERROR                     2215
 250 #define ERR_NO_CORS_FOUND        2216
 251 #define ERR_INVALID_WEBSITE_ROUTING_RULES_ERROR          2217
 252 #define ERR_RATE_LIMITED         2218
 253 #define ERR_POSITION_NOT_EQUAL_TO_LENGTH                 2219
 254 #define ERR_OBJECT_NOT_APPENDABLE                        2220
 255 #define ERR_INVALID_BUCKET_STATE                         2221
 256
 257 #define ERR_BUSY_RESHARDING      2300
 258 #define ERR_NO_SUCH_ENTITY       2301
 259 #define ERR_LIMIT_EXCEEDED       2302
 260
 261 // STS Errors
 262 #define ERR_PACKED_POLICY_TOO_LARGE 2400
 263 #define ERR_INVALID_IDENTITY_TOKEN  2401
 264
 265 #define ERR_NO_SUCH_TAG_SET 2402
 266
 267 #ifndef UINT32_MAX
 268 #define UINT32_MAX (0xffffffffu)
 269 #endif
 270
 271 struct req_state;
 272
 273 typedef void *RGWAccessHandle;
 274
 275 enum RGWIntentEvent {
 276   DEL_OBJ = 0,
 277   DEL_DIR = 1,
 278 };
 279
 280 enum HostStyle {
 281   PathStyle = 0,
 282   VirtualStyle = 1,
 283 };
 284
 285 /** Store error returns for output at a different point in the program */
 286 struct rgw_err {
 287   rgw_err();
 288   void clear();
 289   bool is_clear() const;
 290   bool is_err() const;
 291   friend std::ostream& operator<<(std::ostream& oss, const rgw_err &err);
 292
 293   int http_ret;
 294   int ret;
 295   std::string err_code;
 296   std::string message;
 297 };
 298
 299 /* Helper class used for RGWHTTPArgs parsing */
 300 class NameVal
 301 {
 302    const std::string str;
 303    std::string name;
 304    std::string val;
 305  public:
 306     explicit NameVal(const std::string& nv) : str(nv) {}
 307
 308     int parse();
 309
 310     std::string& get_name() { return name; }
 311     std::string& get_val() { return val; }
 312 };
 313
 314 /** Stores the XML arguments associated with the HTTP request in req_state*/
 315 class RGWHTTPArgs {
 316   std::string str, empty_str;
 317   std::map<std::string, std::string> val_map;
 318   std::map<std::string, std::string> sys_val_map;
 319   std::map<std::string, std::string> sub_resources;
 320   bool has_resp_modifier = false;
 321   bool admin_subresource_added = false;
 322  public:
 323   RGWHTTPArgs() = default;
 324   explicit RGWHTTPArgs(const std::string& s, const DoutPrefixProvider *dpp) {
 325       set(s);
 326       parse(dpp);
 327   }
 328
 329   /** Set the arguments; as received */
 330   void set(const std::string& s) {
 331     has_resp_modifier = false;
 332     val_map.clear();
 333     sub_resources.clear();
 334     str = s;
 335   }
 336   /** parse the received arguments */
 337   int parse(const DoutPrefixProvider *dpp);
 338   void append(const std::string& name, const string& val);
 339   /** Get the value for a specific argument parameter */
 340   const string& get(const std::string& name, bool *exists = NULL) const;
 341   boost::optional<const std::string&>
 342   get_optional(const std::string& name) const;
 343   int get_bool(const std::string& name, bool *val, bool *exists);
 344   int get_bool(const char *name, bool *val, bool *exists);
 345   void get_bool(const char *name, bool *val, bool def_val);
 346   int get_int(const char *name, int *val, int def_val);
 347
 348   /** Get the value for specific system argument parameter */
 349   std::string sys_get(const std::string& name, bool *exists = nullptr) const;
 350
 351   /** see if a parameter is contained in this RGWHTTPArgs */
 352   bool exists(const char *name) const {
 353     return (val_map.find(name) != std::end(val_map));
 354   }
 355   bool sub_resource_exists(const char *name) const {
 356     return (sub_resources.find(name) != std::end(sub_resources));
 357   }
 358   std::map<std::string, std::string>& get_params() {
 359     return val_map;
 360   }
 361   const std::map<std::string, std::string>& get_sub_resources() const {
 362     return sub_resources;
 363   }
 364   unsigned get_num_params() const {
 365     return val_map.size();
 366   }
 367   bool has_response_modifier() const {
 368     return has_resp_modifier;
 369   }
 370   void set_system() { /* make all system params visible */
 371     std::map<std::string, std::string>::iterator iter;
 372     for (iter = sys_val_map.begin(); iter != sys_val_map.end(); ++iter) {
 373       val_map[iter->first] = iter->second;
 374     }
 375   }
 376   const std::string& get_str() {
 377     return str;
 378   }
 379 }; // RGWHTTPArgs
 380
 381 const char *rgw_conf_get(const map<string, string, ltstr_nocase>& conf_map, const char *name, const char *def_val);
 382 int rgw_conf_get_int(const map<string, string, ltstr_nocase>& conf_map, const char *name, int def_val);
 383 bool rgw_conf_get_bool(const map<string, string, ltstr_nocase>& conf_map, const char *name, bool def_val);
 384
 385 class RGWEnv;
 386
 387 class RGWConf {
 388   friend class RGWEnv;
 389   int enable_ops_log;
 390   int enable_usage_log;
 391   uint8_t defer_to_bucket_acls;
 392   void init(CephContext *cct);
 393 public:
 394   RGWConf()
 395     : enable_ops_log(1),
 396       enable_usage_log(1),
 397       defer_to_bucket_acls(0) {
 398   }
 399 };
 400
 401 class RGWEnv {
 402   std::map<string, string, ltstr_nocase> env_map;
 403   RGWConf conf;
 404 public:
 405   void init(CephContext *cct);
 406   void init(CephContext *cct, char **envp);
 407   void set(std::string name, std::string val);
 408   const char *get(const char *name, const char *def_val = nullptr) const;
 409   int get_int(const char *name, int def_val = 0) const;
 410   bool get_bool(const char *name, bool def_val = 0);
 411   size_t get_size(const char *name, size_t def_val = 0) const;
 412   bool exists(const char *name) const;
 413   bool exists_prefix(const char *prefix) const;
 414   void remove(const char *name);
 415   const std::map<string, string, ltstr_nocase>& get_map() const { return env_map; }
 416   int get_enable_ops_log() const {
 417     return conf.enable_ops_log;
 418   }
 419
 420   int get_enable_usage_log() const {
 421     return conf.enable_usage_log;
 422   }
 423
 424   int get_defer_to_bucket_acls() const {
 425     return conf.defer_to_bucket_acls;
 426   }
 427 };
 428
 429 // return true if the connection is secure. this either means that the
 430 // connection arrived via ssl, or was forwarded as https by a trusted proxy
 431 bool rgw_transport_is_secure(CephContext *cct, const RGWEnv& env);
 432
 433 enum http_op {
 434   OP_GET,
 435   OP_PUT,
 436   OP_DELETE,
 437   OP_HEAD,
 438   OP_POST,
 439   OP_COPY,
 440   OP_OPTIONS,
 441   OP_UNKNOWN,
 442 };
 443
 444 class RGWAccessControlPolicy;
 445 class JSONObj;
 446
 447 struct RGWAccessKey {
 448   string id; // AccessKey
 449   string key; // SecretKey
 450   string subuser;
 451
 452   RGWAccessKey() {}
 453   RGWAccessKey(std::string _id, std::string _key)
 454     : id(std::move(_id)), key(std::move(_key)) {}
 455
 456   void encode(bufferlist& bl) const {
 457     ENCODE_START(2, 2, bl);
 458     encode(id, bl);
 459     encode(key, bl);
 460     encode(subuser, bl);
 461     ENCODE_FINISH(bl);
 462   }
 463
 464   void decode(bufferlist::const_iterator& bl) {
 465      DECODE_START_LEGACY_COMPAT_LEN_32(2, 2, 2, bl);
 466      decode(id, bl);
 467      decode(key, bl);
 468      decode(subuser, bl);
 469      DECODE_FINISH(bl);
 470   }
 471   void dump(Formatter *f) const;
 472   void dump_plain(Formatter *f) const;
 473   void dump(Formatter *f, const string& user, bool swift) const;
 474   static void generate_test_instances(list<RGWAccessKey*>& o);
 475
 476   void decode_json(JSONObj *obj);
 477   void decode_json(JSONObj *obj, bool swift);
 478 };
 479 WRITE_CLASS_ENCODER(RGWAccessKey)
 480
 481 struct RGWSubUser {
 482   string name;
 483   uint32_t perm_mask;
 484
 485   RGWSubUser() : perm_mask(0) {}
 486   void encode(bufferlist& bl) const {
 487     ENCODE_START(2, 2, bl);
 488     encode(name, bl);
 489     encode(perm_mask, bl);
 490     ENCODE_FINISH(bl);
 491   }
 492
 493   void decode(bufferlist::const_iterator& bl) {
 494      DECODE_START_LEGACY_COMPAT_LEN_32(2, 2, 2, bl);
 495      decode(name, bl);
 496      decode(perm_mask, bl);
 497      DECODE_FINISH(bl);
 498   }
 499   void dump(Formatter *f) const;
 500   void dump(Formatter *f, const string& user) const;
 501   static void generate_test_instances(list<RGWSubUser*>& o);
 502
 503   void decode_json(JSONObj *obj);
 504 };
 505 WRITE_CLASS_ENCODER(RGWSubUser)
 506
 507 class RGWUserCaps
 508 {
 509   map<string, uint32_t> caps;
 510
 511   int get_cap(const string& cap, string& type, uint32_t *perm);
 512   int add_cap(const string& cap);
 513   int remove_cap(const string& cap);
 514 public:
 515   static int parse_cap_perm(const string& str, uint32_t *perm);
 516   int add_from_string(const string& str);
 517   int remove_from_string(const string& str);
 518
 519   void encode(bufferlist& bl) const {
 520      ENCODE_START(1, 1, bl);
 521      encode(caps, bl);
 522      ENCODE_FINISH(bl);
 523   }
 524   void decode(bufferlist::const_iterator& bl) {
 525      DECODE_START(1, bl);
 526      decode(caps, bl);
 527      DECODE_FINISH(bl);
 528   }
 529   int check_cap(const string& cap, uint32_t perm) const;
 530   bool is_valid_cap_type(const string& tp);
 531   void dump(Formatter *f) const;
 532   void dump(Formatter *f, const char *name) const;
 533
 534   void decode_json(JSONObj *obj);
 535 };
 536 WRITE_CLASS_ENCODER(RGWUserCaps)
 537
 538 void encode_json(const char *name, const obj_version& v, Formatter *f);
 539 void encode_json(const char *name, const RGWUserCaps& val, Formatter *f);
 540
 541 void decode_json_obj(obj_version& v, JSONObj *obj);
 542
 543
 544
 545 enum RGWIdentityType
 546 {
 547   TYPE_NONE=0,
 548   TYPE_RGW=1,
 549   TYPE_KEYSTONE=2,
 550   TYPE_LDAP=3,
 551   TYPE_ROLE=4,
 552   TYPE_WEB=5,
 553 };
 554
 555 static string RGW_STORAGE_CLASS_STANDARD = "STANDARD";
 556
 557 struct rgw_placement_rule {
 558   std::string name;
 559   std::string storage_class;
 560
 561   rgw_placement_rule() {}
 562   rgw_placement_rule(const string& _n, const string& _sc) : name(_n), storage_class(_sc) {}
 563   rgw_placement_rule(const rgw_placement_rule& _r, const string& _sc) : name(_r.name) {
 564     if (!_sc.empty()) {
 565       storage_class = _sc;
 566     } else {
 567       storage_class = _r.storage_class;
 568     }
 569   }
 570
 571   bool empty() const {
 572     return name.empty() && storage_class.empty();
 573   }
 574
 575   void inherit_from(const rgw_placement_rule& r) {
 576     if (name.empty()) {
 577       name = r.name;
 578     }
 579     if (storage_class.empty()) {
 580       storage_class = r.storage_class;
 581     }
 582   }
 583
 584   void clear() {
 585     name.clear();
 586     storage_class.clear();
 587   }
 588
 589   void init(const string& n, const string& c) {
 590     name = n;
 591     storage_class = c;
 592   }
 593
 594   static const string& get_canonical_storage_class(const string& storage_class) {
 595     if (storage_class.empty()) {
 596       return RGW_STORAGE_CLASS_STANDARD;
 597     }
 598     return storage_class;
 599   }
 600
 601   const string& get_storage_class() const {
 602     return get_canonical_storage_class(storage_class);
 603   }
 604
 605   int compare(const rgw_placement_rule& r) const {
 606     int c = name.compare(r.name);
 607     if (c != 0) {
 608       return c;
 609     }
 610     return get_storage_class().compare(r.get_storage_class());
 611   }
 612
 613   bool operator==(const rgw_placement_rule& r) const {
 614     return (name == r.name &&
 615             get_storage_class() == r.get_storage_class());
 616   }
 617
 618   bool operator!=(const rgw_placement_rule& r) const {
 619     return !(*this == r);
 620   }
 621
 622   void encode(bufferlist& bl) const {
 623     /* no ENCODE_START/END due to backward compatibility */
 624     std::string s = to_str();
 625     ceph::encode(s, bl);
 626   }
 627
 628   void decode(bufferlist::const_iterator& bl) {
 629     std::string s;
 630     ceph::decode(s, bl);
 631     from_str(s);
 632   }
 633
 634   std::string to_str() const {
 635     if (standard_storage_class()) {
 636       return name;
 637     }
 638     return to_str_explicit();
 639   }
 640
 641   std::string to_str_explicit() const {
 642     return name + "/" + storage_class;
 643   }
 644
 645   void from_str(const std::string& s) {
 646     size_t pos = s.find("/");
 647     if (pos == std::string::npos) {
 648       name = s;
 649       storage_class.clear();
 650       return;
 651     }
 652     name = s.substr(0, pos);
 653     storage_class = s.substr(pos + 1);
 654   }
 655
 656   bool standard_storage_class() const {
 657     return storage_class.empty() || storage_class == RGW_STORAGE_CLASS_STANDARD;
 658   }
 659 };
 660 WRITE_CLASS_ENCODER(rgw_placement_rule)
 661
 662 void encode_json(const char *name, const rgw_placement_rule& val, ceph::Formatter *f);
 663 void decode_json_obj(rgw_placement_rule& v, JSONObj *obj);
 664
 665 inline ostream& operator<<(ostream& out, const rgw_placement_rule& rule) {
 666   return out << rule.to_str();
 667 }
 668 struct RGWUserInfo
 669 {
 670   rgw_user user_id;
 671   string display_name;
 672   string user_email;
 673   map<string, RGWAccessKey> access_keys;
 674   map<string, RGWAccessKey> swift_keys;
 675   map<string, RGWSubUser> subusers;
 676   __u8 suspended;
 677   int32_t max_buckets;
 678   uint32_t op_mask;
 679   RGWUserCaps caps;
 680   __u8 admin;
 681   __u8 system;
 682   rgw_placement_rule default_placement;
 683   list<string> placement_tags;
 684   RGWQuotaInfo bucket_quota;
 685   map<int, string> temp_url_keys;
 686   RGWQuotaInfo user_quota;
 687   uint32_t type;
 688   set<string> mfa_ids;
 689   string assumed_role_arn;
 690
 691   RGWUserInfo()
 692     : suspended(0),
 693       max_buckets(RGW_DEFAULT_MAX_BUCKETS),
 694       op_mask(RGW_OP_TYPE_ALL),
 695       admin(0),
 696       system(0),
 697       type(TYPE_NONE) {
 698   }
 699
 700   RGWAccessKey* get_key(const string& access_key) {
 701     if (access_keys.empty())
 702       return nullptr;
 703
 704     auto k = access_keys.find(access_key);
 705     if (k == access_keys.end())
 706       return nullptr;
 707     else
 708       return &(k->second);
 709   }
 710
 711   void encode(bufferlist& bl) const {
 712      ENCODE_START(22, 9, bl);
 713      encode((uint64_t)0, bl); // old auid
 714      string access_key;
 715      string secret_key;
 716      if (!access_keys.empty()) {
 717        map<string, RGWAccessKey>::const_iterator iter = access_keys.begin();
 718        const RGWAccessKey& k = iter->second;
 719        access_key = k.id;
 720        secret_key = k.key;
 721      }
 722      encode(access_key, bl);
 723      encode(secret_key, bl);
 724      encode(display_name, bl);
 725      encode(user_email, bl);
 726      string swift_name;
 727      string swift_key;
 728      if (!swift_keys.empty()) {
 729        map<string, RGWAccessKey>::const_iterator iter = swift_keys.begin();
 730        const RGWAccessKey& k = iter->second;
 731        swift_name = k.id;
 732        swift_key = k.key;
 733      }
 734      encode(swift_name, bl);
 735      encode(swift_key, bl);
 736      encode(user_id.id, bl);
 737      encode(access_keys, bl);
 738      encode(subusers, bl);
 739      encode(suspended, bl);
 740      encode(swift_keys, bl);
 741      encode(max_buckets, bl);
 742      encode(caps, bl);
 743      encode(op_mask, bl);
 744      encode(system, bl);
 745      encode(default_placement, bl);
 746      encode(placement_tags, bl);
 747      encode(bucket_quota, bl);
 748      encode(temp_url_keys, bl);
 749      encode(user_quota, bl);
 750      encode(user_id.tenant, bl);
 751      encode(admin, bl);
 752      encode(type, bl);
 753      encode(mfa_ids, bl);
 754      encode(assumed_role_arn, bl);
 755      encode(user_id.ns, bl);
 756      ENCODE_FINISH(bl);
 757   }
 758   void decode(bufferlist::const_iterator& bl) {
 759      DECODE_START_LEGACY_COMPAT_LEN_32(22, 9, 9, bl);
 760      if (struct_v >= 2) {
 761        uint64_t old_auid;
 762        decode(old_auid, bl);
 763      }
 764      string access_key;
 765      string secret_key;
 766     decode(access_key, bl);
 767     decode(secret_key, bl);
 768     if (struct_v < 6) {
 769       RGWAccessKey k;
 770       k.id = access_key;
 771       k.key = secret_key;
 772       access_keys[access_key] = k;
 773     }
 774     decode(display_name, bl);
 775     decode(user_email, bl);
 776     /* We populate swift_keys map later nowadays, but we have to decode. */
 777     string swift_name;
 778     string swift_key;
 779     if (struct_v >= 3) decode(swift_name, bl);
 780     if (struct_v >= 4) decode(swift_key, bl);
 781     if (struct_v >= 5)
 782       decode(user_id.id, bl);
 783     else
 784       user_id.id = access_key;
 785     if (struct_v >= 6) {
 786       decode(access_keys, bl);
 787       decode(subusers, bl);
 788     }
 789     suspended = 0;
 790     if (struct_v >= 7) {
 791       decode(suspended, bl);
 792     }
 793     if (struct_v >= 8) {
 794       decode(swift_keys, bl);
 795     }
 796     if (struct_v >= 10) {
 797       decode(max_buckets, bl);
 798     } else {
 799       max_buckets = RGW_DEFAULT_MAX_BUCKETS;
 800     }
 801     if (struct_v >= 11) {
 802       decode(caps, bl);
 803     }
 804     if (struct_v >= 12) {
 805       decode(op_mask, bl);
 806     } else {
 807       op_mask = RGW_OP_TYPE_ALL;
 808     }
 809     if (struct_v >= 13) {
 810       decode(system, bl);
 811       decode(default_placement, bl);
 812       decode(placement_tags, bl); /* tags of allowed placement rules */
 813     }
 814     if (struct_v >= 14) {
 815       decode(bucket_quota, bl);
 816     }
 817     if (struct_v >= 15) {
 818      decode(temp_url_keys, bl);
 819     }
 820     if (struct_v >= 16) {
 821       decode(user_quota, bl);
 822     }
 823     if (struct_v >= 17) {
 824       decode(user_id.tenant, bl);
 825     } else {
 826       user_id.tenant.clear();
 827     }
 828     if (struct_v >= 18) {
 829       decode(admin, bl);
 830     }
 831     if (struct_v >= 19) {
 832       decode(type, bl);
 833     }
 834     if (struct_v >= 20) {
 835       decode(mfa_ids, bl);
 836     }
 837     if (struct_v >= 21) {
 838       decode(assumed_role_arn, bl);
 839     }
 840     if (struct_v >= 22) {
 841       decode(user_id.ns, bl);
 842     } else {
 843       user_id.ns.clear();
 844     }
 845     DECODE_FINISH(bl);
 846   }
 847   void dump(Formatter *f) const;
 848   static void generate_test_instances(list<RGWUserInfo*>& o);
 849
 850   void decode_json(JSONObj *obj);
 851 };
 852 WRITE_CLASS_ENCODER(RGWUserInfo)
 853
 854 struct rgw_raw_obj {
 855   rgw_pool pool;
 856   std::string oid;
 857   std::string loc;
 858
 859   rgw_raw_obj() {}
 860   rgw_raw_obj(const rgw_pool& _pool, const std::string& _oid) {
 861     init(_pool, _oid);
 862   }
 863   rgw_raw_obj(const rgw_pool& _pool, const std::string& _oid, const string& _loc) : loc(_loc) {
 864     init(_pool, _oid);
 865   }
 866
 867   void init(const rgw_pool& _pool, const std::string& _oid) {
 868     pool = _pool;
 869     oid = _oid;
 870   }
 871
 872   bool empty() const {
 873     return oid.empty();
 874   }
 875
 876   void encode(bufferlist& bl) const {
 877      ENCODE_START(6, 6, bl);
 878     encode(pool, bl);
 879     encode(oid, bl);
 880     encode(loc, bl);
 881     ENCODE_FINISH(bl);
 882   }
 883
 884   void decode_from_rgw_obj(bufferlist::const_iterator& bl);
 885
 886   void decode(bufferlist::const_iterator& bl) {
 887     unsigned ofs = bl.get_off();
 888     DECODE_START(6, bl);
 889     if (struct_v < 6) {
 890       /*
 891        * this object was encoded as rgw_obj, prior to rgw_raw_obj been split out of it,
 892        * let's decode it as rgw_obj and convert it
 893        */
 894       bl.seek(ofs);
 895       decode_from_rgw_obj(bl);
 896       return;
 897     }
 898     decode(pool, bl);
 899     decode(oid, bl);
 900     decode(loc, bl);
 901     DECODE_FINISH(bl);
 902   }
 903
 904   bool operator<(const rgw_raw_obj& o) const {
 905     int r = pool.compare(o.pool);
 906     if (r == 0) {
 907       r = oid.compare(o.oid);
 908       if (r == 0) {
 909         r = loc.compare(o.loc);
 910       }
 911     }
 912     return (r < 0);
 913   }
 914
 915   bool operator==(const rgw_raw_obj& o) const {
 916     return (pool == o.pool && oid == o.oid && loc == o.loc);
 917   }
 918
 919   void dump(Formatter *f) const;
 920   void decode_json(JSONObj *obj);
 921 };
 922 WRITE_CLASS_ENCODER(rgw_raw_obj)
 923
 924 inline ostream& operator<<(ostream& out, const rgw_raw_obj& o) {
 925   out << o.pool << ":" << o.oid;
 926   return out;
 927 }
 928
 929 struct rgw_bucket_placement {
 930   rgw_placement_rule placement_rule;
 931   rgw_bucket bucket;
 932
 933   void dump(Formatter *f) const;
 934 };
 935
 936 struct RGWObjVersionTracker {
 937   obj_version read_version;
 938   obj_version write_version;
 939
 940   obj_version *version_for_read() {
 941     return &read_version;
 942   }
 943
 944   obj_version *version_for_write() {
 945     if (write_version.ver == 0)
 946       return NULL;
 947
 948     return &write_version;
 949   }
 950
 951   obj_version *version_for_check() {
 952     if (read_version.ver == 0)
 953       return NULL;
 954
 955     return &read_version;
 956   }
 957
 958   void prepare_op_for_read(librados::ObjectReadOperation *op);
 959   void prepare_op_for_write(librados::ObjectWriteOperation *op);
 960
 961   void apply_write();
 962
 963   void clear() {
 964     read_version = obj_version();
 965     write_version = obj_version();
 966   }
 967
 968   void generate_new_write_ver(CephContext *cct);
 969 };
 970
 971 inline ostream& operator<<(ostream& out, const obj_version &v)
 972 {
 973   out << v.tag << ":" << v.ver;
 974   return out;
 975 }
 976
 977 inline ostream& operator<<(ostream& out, const RGWObjVersionTracker &ot)
 978 {
 979   out << "{r=" << ot.read_version << ",w=" << ot.write_version << "}";
 980   return out;
 981 }
 982
 983 enum RGWBucketFlags {
 984   BUCKET_SUSPENDED = 0x1,
 985   BUCKET_VERSIONED = 0x2,
 986   BUCKET_VERSIONS_SUSPENDED = 0x4,
 987   BUCKET_DATASYNC_DISABLED = 0X8,
 988   BUCKET_MFA_ENABLED = 0X10,
 989   BUCKET_OBJ_LOCK_ENABLED = 0X20,
 990 };
 991
 992 class RGWSI_Zone;
 993
 994 struct RGWBucketInfo {
 995   rgw_bucket bucket;
 996   rgw_user owner;
 997   uint32_t flags{0};
 998   string zonegroup;
 999   ceph::real_time creation_time;
1000   rgw_placement_rule placement_rule;
1001   bool has_instance_obj{false};
1002   RGWObjVersionTracker objv_tracker; /* we don't need to serialize this, for runtime tracking */
1003   RGWQuotaInfo quota;
1004
1005   // layout of bucket index objects
1006   rgw::BucketLayout layout;
1007
1008   // Represents the number of bucket index object shards:
1009   //   - value of 0 indicates there is no sharding (this is by default
1010   //     before this feature is implemented).
1011   //   - value of UINT32_T::MAX indicates this is a blind bucket.
1012
1013   // Represents the shard number for blind bucket.
1014   const static uint32_t NUM_SHARDS_BLIND_BUCKET;
1015
1016   bool requester_pays{false};
1017
1018   bool has_website{false};
1019   RGWBucketWebsiteConf website_conf;
1020
1021   bool swift_versioning{false};
1022   string swift_ver_location;
1023
1024   map<string, uint32_t> mdsearch_config;
1025
1026   // resharding
1027   cls_rgw_reshard_status reshard_status{cls_rgw_reshard_status::NOT_RESHARDING};
1028   string new_bucket_instance_id;
1029
1030   RGWObjectLock obj_lock;
1031
1032   std::optional<rgw_sync_policy_info> sync_policy;
1033
1034   void encode(bufferlist& bl) const;
1035   void decode(bufferlist::const_iterator& bl);
1036
1037   void dump(Formatter *f) const;
1038   static void generate_test_instances(list<RGWBucketInfo*>& o);
1039
1040   void decode_json(JSONObj *obj);
1041
1042   bool versioned() const { return (flags & BUCKET_VERSIONED) != 0; }
1043   int versioning_status() const { return flags & (BUCKET_VERSIONED | BUCKET_VERSIONS_SUSPENDED | BUCKET_MFA_ENABLED); }
1044   bool versioning_enabled() const { return (versioning_status() & (BUCKET_VERSIONED | BUCKET_VERSIONS_SUSPENDED)) == BUCKET_VERSIONED; }
1045   bool mfa_enabled() const { return (versioning_status() & BUCKET_MFA_ENABLED) != 0; }
1046   bool datasync_flag_enabled() const { return (flags & BUCKET_DATASYNC_DISABLED) == 0; }
1047   bool obj_lock_enabled() const { return (flags & BUCKET_OBJ_LOCK_ENABLED) != 0; }
1048
1049   bool has_swift_versioning() const {
1050     /* A bucket may be versioned through one mechanism only. */
1051     return swift_versioning && !versioned();
1052   }
1053
1054   void set_sync_policy(rgw_sync_policy_info&& policy);
1055
1056   bool empty_sync_policy() const;
1057
1058   RGWBucketInfo();
1059   ~RGWBucketInfo();
1060 };
1061 WRITE_CLASS_ENCODER(RGWBucketInfo)
1062
1063 struct RGWBucketEntryPoint
1064 {
1065   rgw_bucket bucket;
1066   rgw_user owner;
1067   ceph::real_time creation_time;
1068   bool linked;
1069
1070   bool has_bucket_info;
1071   RGWBucketInfo old_bucket_info;
1072
1073   RGWBucketEntryPoint() : linked(false), has_bucket_info(false) {}
1074
1075   void encode(bufferlist& bl) const {
1076     ENCODE_START(10, 8, bl);
1077     encode(bucket, bl);
1078     encode(owner.id, bl);
1079     encode(linked, bl);
1080     uint64_t ctime = (uint64_t)real_clock::to_time_t(creation_time);
1081     encode(ctime, bl);
1082     encode(owner, bl);
1083     encode(creation_time, bl);
1084     ENCODE_FINISH(bl);
1085   }
1086   void decode(bufferlist::const_iterator& bl) {
1087     auto orig_iter = bl;
1088     DECODE_START_LEGACY_COMPAT_LEN_32(10, 4, 4, bl);
1089     if (struct_v < 8) {
1090       /* ouch, old entry, contains the bucket info itself */
1091       old_bucket_info.decode(orig_iter);
1092       has_bucket_info = true;
1093       return;
1094     }
1095     has_bucket_info = false;
1096     decode(bucket, bl);
1097     decode(owner.id, bl);
1098     decode(linked, bl);
1099     uint64_t ctime;
1100     decode(ctime, bl);
1101     if (struct_v < 10) {
1102       creation_time = real_clock::from_time_t((time_t)ctime);
1103     }
1104     if (struct_v >= 9) {
1105       decode(owner, bl);
1106     }
1107     if (struct_v >= 10) {
1108       decode(creation_time, bl);
1109     }
1110     DECODE_FINISH(bl);
1111   }
1112
1113   void dump(Formatter *f) const;
1114   void decode_json(JSONObj *obj);
1115   static void generate_test_instances(list<RGWBucketEntryPoint*>& o);
1116 };
1117 WRITE_CLASS_ENCODER(RGWBucketEntryPoint)
1118
1119 struct RGWStorageStats
1120 {
1121   RGWObjCategory category;
1122   uint64_t size;
1123   uint64_t size_rounded;
1124   uint64_t size_utilized{0}; //< size after compression, encryption
1125   uint64_t num_objects;
1126
1127   RGWStorageStats()
1128     : category(RGWObjCategory::None),
1129       size(0),
1130       size_rounded(0),
1131       num_objects(0) {}
1132
1133   void dump(Formatter *f) const;
1134 };
1135
1136 class RGWEnv;
1137
1138 /* Namespaced forward declarations. */
1139 namespace rgw {
1140   namespace auth {
1141     namespace s3 {
1142       class AWSBrowserUploadAbstractor;
1143       class STSEngine;
1144     }
1145     class Completer;
1146   }
1147   namespace io {
1148     class BasicClient;
1149   }
1150 }
1151
1152 using meta_map_t = boost::container::flat_map <std::string, std::string>;
1153
1154 struct req_info {
1155   const RGWEnv *env;
1156   RGWHTTPArgs args;
1157   meta_map_t x_meta_map;
1158
1159   string host;
1160   const char *method;
1161   string script_uri;
1162   string request_uri;
1163   string request_uri_aws4;
1164   string effective_uri;
1165   string request_params;
1166   string domain;
1167   string storage_class;
1168
1169   req_info(CephContext *cct, const RGWEnv *env);
1170   void rebuild_from(req_info& src);
1171   void init_meta_info(const DoutPrefixProvider *dpp, bool *found_bad_meta);
1172 };
1173
1174 typedef cls_rgw_obj_key rgw_obj_index_key;
1175
1176 struct rgw_obj_key {
1177   string name;
1178   string instance;
1179   string ns;
1180
1181   rgw_obj_key() {}
1182   // cppcheck-suppress noExplicitConstructor
1183   rgw_obj_key(const string& n) : name(n) {}
1184   rgw_obj_key(const string& n, const string& i) : name(n), instance(i) {}
1185   rgw_obj_key(const string& n, const string& i, const string& _ns) : name(n), instance(i), ns(_ns) {}
1186
1187   rgw_obj_key(const rgw_obj_index_key& k) {
1188     parse_index_key(k.name, &name, &ns);
1189     instance = k.instance;
1190   }
1191
1192   static void parse_index_key(const string& key, string *name, string *ns) {
1193     if (key[0] != '_') {
1194       *name = key;
1195       ns->clear();
1196       return;
1197     }
1198     if (key[1] == '_') {
1199       *name = key.substr(1);
1200       ns->clear();
1201       return;
1202     }
1203     ssize_t pos = key.find('_', 1);
1204     if (pos < 0) {
1205       /* shouldn't happen, just use key */
1206       *name = key;
1207       ns->clear();
1208       return;
1209     }
1210
1211     *name = key.substr(pos + 1);
1212     *ns = key.substr(1, pos -1);
1213   }
1214
1215   void set(const string& n) {
1216     name = n;
1217     instance.clear();
1218     ns.clear();
1219   }
1220
1221   void set(const string& n, const string& i) {
1222     name = n;
1223     instance = i;
1224     ns.clear();
1225   }
1226
1227   void set(const string& n, const string& i, const string& _ns) {
1228     name = n;
1229     instance = i;
1230     ns = _ns;
1231   }
1232
1233   bool set(const rgw_obj_index_key& index_key) {
1234     if (!parse_raw_oid(index_key.name, this)) {
1235       return false;
1236     }
1237     instance = index_key.instance;
1238     return true;
1239   }
1240
1241   void set_instance(const string& i) {
1242     instance = i;
1243   }
1244
1245   const string& get_instance() const {
1246     return instance;
1247   }
1248
1249   void set_ns(const std::string& _ns) {
1250     ns = _ns;
1251   }
1252
1253   const std::string& get_ns() const {
1254     return ns;
1255   }
1256
1257   string get_index_key_name() const {
1258     if (ns.empty()) {
1259       if (name.size() < 1 || name[0] != '_') {
1260         return name;
1261       }
1262       return string("_") + name;
1263     };
1264
1265     char buf[ns.size() + 16];
1266     snprintf(buf, sizeof(buf), "_%s_", ns.c_str());
1267     return string(buf) + name;
1268   };
1269
1270   void get_index_key(rgw_obj_index_key *key) const {
1271     key->name = get_index_key_name();
1272     key->instance = instance;
1273   }
1274
1275   string get_loc() const {
1276     /*
1277      * For backward compatibility. Older versions used to have object locator on all objects,
1278      * however, the name was the effective object locator. This had the same effect as not
1279      * having object locator at all for most objects but the ones that started with underscore as
1280      * these were escaped.
1281      */
1282     if (name[0] == '_' && ns.empty()) {
1283       return name;
1284     }
1285
1286     return string();
1287   }
1288
1289   bool empty() const {
1290     return name.empty();
1291   }
1292
1293   bool have_null_instance() const {
1294     return instance == "null";
1295   }
1296
1297   bool have_instance() const {
1298     return !instance.empty();
1299   }
1300
1301   bool need_to_encode_instance() const {
1302     return have_instance() && !have_null_instance();
1303   }
1304
1305   string get_oid() const {
1306     if (ns.empty() && !need_to_encode_instance()) {
1307       if (name.size() < 1 || name[0] != '_') {
1308         return name;
1309       }
1310       return string("_") + name;
1311     }
1312
1313     string oid = "_";
1314     oid.append(ns);
1315     if (need_to_encode_instance()) {
1316       oid.append(string(":") + instance);
1317     }
1318     oid.append("_");
1319     oid.append(name);
1320     return oid;
1321   }
1322
1323   bool operator==(const rgw_obj_key& k) const {
1324     return (name.compare(k.name) == 0) &&
1325            (instance.compare(k.instance) == 0);
1326   }
1327
1328   bool operator<(const rgw_obj_key& k) const {
1329     int r = name.compare(k.name);
1330     if (r == 0) {
1331       r = instance.compare(k.instance);
1332     }
1333     return (r < 0);
1334   }
1335
1336   bool operator<=(const rgw_obj_key& k) const {
1337     return !(k < *this);
1338   }
1339
1340   static void parse_ns_field(string& ns, string& instance) {
1341     int pos = ns.find(':');
1342     if (pos >= 0) {
1343       instance = ns.substr(pos + 1);
1344       ns = ns.substr(0, pos);
1345     } else {
1346       instance.clear();
1347     }
1348   }
1349
1350   // takes an oid and parses out the namespace (ns), name, and
1351   // instance
1352   static bool parse_raw_oid(const string& oid, rgw_obj_key *key) {
1353     key->instance.clear();
1354     key->ns.clear();
1355     if (oid[0] != '_') {
1356       key->name = oid;
1357       return true;
1358     }
1359
1360     if (oid.size() >= 2 && oid[1] == '_') {
1361       key->name = oid.substr(1);
1362       return true;
1363     }
1364
1365     if (oid.size() < 3) // for namespace, min size would be 3: _x_
1366       return false;
1367
1368     size_t pos = oid.find('_', 2); // oid must match ^_[^_].+$
1369     if (pos == string::npos)
1370       return false;
1371
1372     key->ns = oid.substr(1, pos - 1);
1373     parse_ns_field(key->ns, key->instance);
1374
1375     key->name = oid.substr(pos + 1);
1376     return true;
1377   }
1378
1379   /**
1380    * Translate a namespace-mangled object name to the user-facing name
1381    * existing in the given namespace.
1382    *
1383    * If the object is part of the given namespace, it returns true
1384    * and cuts down the name to the unmangled version. If it is not
1385    * part of the given namespace, it returns false.
1386    */
1387   static bool oid_to_key_in_ns(const string& oid, rgw_obj_key *key, const string& ns) {
1388     bool ret = parse_raw_oid(oid, key);
1389     if (!ret) {
1390       return ret;
1391     }
1392
1393     return (ns == key->ns);
1394   }
1395
1396   /**
1397    * Given a mangled object name and an empty namespace string, this
1398    * function extracts the namespace into the string and sets the object
1399    * name to be the unmangled version.
1400    *
1401    * It returns true after successfully doing so, or
1402    * false if it fails.
1403    */
1404   static bool strip_namespace_from_name(string& name, string& ns, string& instance) {
1405     ns.clear();
1406     instance.clear();
1407     if (name[0] != '_') {
1408       return true;
1409     }
1410
1411     size_t pos = name.find('_', 1);
1412     if (pos == string::npos) {
1413       return false;
1414     }
1415
1416     if (name[1] == '_') {
1417       name = name.substr(1);
1418       return true;
1419     }
1420
1421     size_t period_pos = name.find('.');
1422     if (period_pos < pos) {
1423       return false;
1424     }
1425
1426     ns = name.substr(1, pos-1);
1427     name = name.substr(pos+1, string::npos);
1428
1429     parse_ns_field(ns, instance);
1430     return true;
1431   }
1432
1433   void encode(bufferlist& bl) const {
1434     ENCODE_START(2, 1, bl);
1435     encode(name, bl);
1436     encode(instance, bl);
1437     encode(ns, bl);
1438     ENCODE_FINISH(bl);
1439   }
1440   void decode(bufferlist::const_iterator& bl) {
1441     DECODE_START(2, bl);
1442     decode(name, bl);
1443     decode(instance, bl);
1444     if (struct_v >= 2) {
1445       decode(ns, bl);
1446     }
1447     DECODE_FINISH(bl);
1448   }
1449   void dump(Formatter *f) const;
1450   void decode_json(JSONObj *obj);
1451
1452   string to_str() const {
1453     if (instance.empty()) {
1454       return name;
1455     }
1456     char buf[name.size() + instance.size() + 16];
1457     snprintf(buf, sizeof(buf), "%s[%s]", name.c_str(), instance.c_str());
1458     return buf;
1459   }
1460 };
1461 WRITE_CLASS_ENCODER(rgw_obj_key)
1462
1463 inline ostream& operator<<(ostream& out, const rgw_obj_key &o) {
1464   return out << o.to_str();
1465 }
1466
1467 inline ostream& operator<<(ostream& out, const rgw_obj_index_key &o) {
1468   if (o.instance.empty()) {
1469     return out << o.name;
1470   } else {
1471     return out << o.name << "[" << o.instance << "]";
1472   }
1473 }
1474
1475 struct req_init_state {
1476   /* Keeps [[tenant]:]bucket until we parse the token. */
1477   string url_bucket;
1478   string src_bucket;
1479 };
1480
1481 #include "rgw_auth.h"
1482
1483 class RGWObjectCtx;
1484 class RGWSysObjectCtx;
1485
1486 /** Store all the state necessary to complete and respond to an HTTP request*/
1487 struct req_state : DoutPrefixProvider {
1488   CephContext *cct;
1489   rgw::io::BasicClient *cio{nullptr};
1490   http_op op{OP_UNKNOWN};
1491   RGWOpType op_type{};
1492   bool content_started{false};
1493   int format{0};
1494   ceph::Formatter *formatter{nullptr};
1495   string decoded_uri;
1496   string relative_uri;
1497   const char *length{nullptr};
1498   int64_t content_length{0};
1499   map<string, string> generic_attrs;
1500   rgw_err err;
1501   bool expect_cont{false};
1502   uint64_t obj_size{0};
1503   bool enable_ops_log;
1504   bool enable_usage_log;
1505   uint8_t defer_to_bucket_acls;
1506   uint32_t perm_mask{0};
1507
1508   /* Set once when url_bucket is parsed and not violated thereafter. */
1509   string account_name;
1510
1511   string bucket_tenant;
1512   string bucket_name;
1513
1514   std::unique_ptr<rgw::sal::RGWBucket> bucket;
1515   std::unique_ptr<rgw::sal::RGWObject> object;
1516   string src_tenant_name;
1517   string src_bucket_name;
1518   std::unique_ptr<rgw::sal::RGWObject> src_object;
1519   ACLOwner bucket_owner;
1520   ACLOwner owner;
1521
1522   string zonegroup_name;
1523   string zonegroup_endpoint;
1524   string bucket_instance_id;
1525   int bucket_instance_shard_id{-1};
1526   string redirect_zone_endpoint;
1527
1528   string redirect;
1529
1530   real_time bucket_mtime;
1531   std::map<std::string, ceph::bufferlist> bucket_attrs;
1532   bool bucket_exists{false};
1533   rgw_placement_rule dest_placement;
1534
1535   bool has_bad_meta{false};
1536
1537   std::unique_ptr<rgw::sal::RGWUser> user;
1538
1539   struct {
1540     /* TODO(rzarzynski): switch out to the static_ptr for both members. */
1541
1542     /* Object having the knowledge about an authenticated identity and allowing
1543      * to apply it during the authorization phase (verify_permission() methods
1544      * of a given RGWOp). Thus, it bounds authentication and authorization steps
1545      * through a well-defined interface. For more details, see rgw_auth.h. */
1546     std::unique_ptr<rgw::auth::Identity> identity;
1547
1548     std::shared_ptr<rgw::auth::Completer> completer;
1549
1550     /* A container for credentials of the S3's browser upload. It's necessary
1551      * because: 1) the ::authenticate() method of auth engines and strategies
1552      * take req_state only; 2) auth strategies live much longer than RGWOps -
1553      * there is no way to pass additional data dependencies through ctors. */
1554     class {
1555       /* Writer. */
1556       friend class RGWPostObj_ObjStore_S3;
1557       /* Reader. */
1558       friend class rgw::auth::s3::AWSBrowserUploadAbstractor;
1559       friend class rgw::auth::s3::STSEngine;
1560
1561       std::string access_key;
1562       std::string signature;
1563       std::string x_amz_algorithm;
1564       std::string x_amz_credential;
1565       std::string x_amz_date;
1566       std::string x_amz_security_token;
1567       ceph::bufferlist encoded_policy;
1568     } s3_postobj_creds;
1569   } auth;
1570
1571   std::unique_ptr<RGWAccessControlPolicy> user_acl;
1572   std::unique_ptr<RGWAccessControlPolicy> bucket_acl;
1573   std::unique_ptr<RGWAccessControlPolicy> object_acl;
1574
1575   rgw::IAM::Environment env;
1576   boost::optional<rgw::IAM::Policy> iam_policy;
1577   boost::optional<PublicAccessBlockConfiguration> bucket_access_conf;
1578   vector<rgw::IAM::Policy> iam_user_policies;
1579
1580   /* Is the request made by an user marked as a system one?
1581    * Being system user means we also have the admin status. */
1582   bool system_request{false};
1583
1584   string canned_acl;
1585   bool has_acl_header{false};
1586   bool local_source{false}; /* source is local */
1587
1588   int prot_flags{0};
1589
1590   /* Content-Disposition override for TempURL of Swift API. */
1591   struct {
1592     string override;
1593     string fallback;
1594   } content_disp;
1595
1596   string host_id;
1597
1598   req_info info;
1599   req_init_state init_state;
1600
1601   using Clock = ceph::coarse_real_clock;
1602   Clock::time_point time;
1603
1604   Clock::duration time_elapsed() const { return Clock::now() - time; }
1605
1606   RGWObjectCtx *obj_ctx{nullptr};
1607   RGWSysObjectCtx *sysobj_ctx{nullptr};
1608   string dialect;
1609   string req_id;
1610   string trans_id;
1611   uint64_t id;
1612
1613   RGWObjTags tagset;
1614
1615   bool mfa_verified{false};
1616
1617   /// optional coroutine context
1618   optional_yield yield{null_yield};
1619
1620   //token claims from STS token for ops log (can be used for Keystone token also)
1621   std::vector<string> token_claims;
1622
1623   req_state(CephContext* _cct, RGWEnv* e, uint64_t id);
1624   ~req_state();
1625
1626
1627   void set_user(std::unique_ptr<rgw::sal::RGWUser>& u) { user.swap(u); }
1628   bool is_err() const { return err.is_err(); }
1629
1630   // implements DoutPrefixProvider
1631   std::ostream& gen_prefix(std::ostream& out) const override;
1632   CephContext* get_cct() const override { return cct; }
1633   unsigned get_subsys() const override { return ceph_subsys_rgw; }
1634 };
1635
1636 void set_req_state_err(struct req_state*, int);
1637 void set_req_state_err(struct req_state*, int, const string&);
1638 void set_req_state_err(struct rgw_err&, int, const int);
1639 void dump(struct req_state*);
1640
1641 /** Store basic data on bucket */
1642 struct RGWBucketEnt {
1643   rgw_bucket bucket;
1644   size_t size;
1645   size_t size_rounded;
1646   ceph::real_time creation_time;
1647   uint64_t count;
1648
1649   /* The placement_rule is necessary to calculate per-storage-policy statics
1650    * of the Swift API. Although the info available in RGWBucketInfo, we need
1651    * to duplicate it here to not affect the performance of buckets listing. */
1652   rgw_placement_rule placement_rule;
1653
1654   RGWBucketEnt()
1655     : size(0),
1656       size_rounded(0),
1657       count(0) {
1658   }
1659   RGWBucketEnt(const RGWBucketEnt&) = default;
1660   RGWBucketEnt(RGWBucketEnt&&) = default;
1661   explicit RGWBucketEnt(const rgw_user& u, cls_user_bucket_entry&& e)
1662     : bucket(u, std::move(e.bucket)),
1663       size(e.size),
1664       size_rounded(e.size_rounded),
1665       creation_time(e.creation_time),
1666       count(e.count) {
1667   }
1668
1669   RGWBucketEnt& operator=(const RGWBucketEnt&) = default;
1670
1671   void convert(cls_user_bucket_entry *b) const {
1672     bucket.convert(&b->bucket);
1673     b->size = size;
1674     b->size_rounded = size_rounded;
1675     b->creation_time = creation_time;
1676     b->count = count;
1677   }
1678
1679   void encode(bufferlist& bl) const {
1680     ENCODE_START(7, 5, bl);
1681     uint64_t s = size;
1682     __u32 mt = ceph::real_clock::to_time_t(creation_time);
1683     string empty_str;  // originally had the bucket name here, but we encode bucket later
1684     encode(empty_str, bl);
1685     encode(s, bl);
1686     encode(mt, bl);
1687     encode(count, bl);
1688     encode(bucket, bl);
1689     s = size_rounded;
1690     encode(s, bl);
1691     encode(creation_time, bl);
1692     encode(placement_rule, bl);
1693     ENCODE_FINISH(bl);
1694   }
1695   void decode(bufferlist::const_iterator& bl) {
1696     DECODE_START_LEGACY_COMPAT_LEN(7, 5, 5, bl);
1697     __u32 mt;
1698     uint64_t s;
1699     string empty_str;  // backward compatibility
1700     decode(empty_str, bl);
1701     decode(s, bl);
1702     decode(mt, bl);
1703     size = s;
1704     if (struct_v < 6) {
1705       creation_time = ceph::real_clock::from_time_t(mt);
1706     }
1707     if (struct_v >= 2)
1708       decode(count, bl);
1709     if (struct_v >= 3)
1710       decode(bucket, bl);
1711     if (struct_v >= 4)
1712       decode(s, bl);
1713     size_rounded = s;
1714     if (struct_v >= 6)
1715       decode(creation_time, bl);
1716     if (struct_v >= 7)
1717       decode(placement_rule, bl);
1718     DECODE_FINISH(bl);
1719   }
1720   void dump(Formatter *f) const;
1721   static void generate_test_instances(list<RGWBucketEnt*>& o);
1722 };
1723 WRITE_CLASS_ENCODER(RGWBucketEnt)
1724
1725 struct rgw_obj {
1726   rgw_bucket bucket;
1727   rgw_obj_key key;
1728
1729   bool in_extra_data{false}; /* in-memory only member, does not serialize */
1730
1731   // Represents the hash index source for this object once it is set (non-empty)
1732   std::string index_hash_source;
1733
1734   rgw_obj() {}
1735   rgw_obj(const rgw_bucket& b, const std::string& name) : bucket(b), key(name) {}
1736   rgw_obj(const rgw_bucket& b, const rgw_obj_key& k) : bucket(b), key(k) {}
1737   rgw_obj(const rgw_bucket& b, const rgw_obj_index_key& k) : bucket(b), key(k) {}
1738
1739   void init(const rgw_bucket& b, const std::string& name) {
1740     bucket = b;
1741     key.set(name);
1742   }
1743   void init(const rgw_bucket& b, const std::string& name, const string& i, const string& n) {
1744     bucket = b;
1745     key.set(name, i, n);
1746   }
1747   void init_ns(const rgw_bucket& b, const std::string& name, const string& n) {
1748     bucket = b;
1749     key.name = name;
1750     key.instance.clear();
1751     key.ns = n;
1752   }
1753
1754   bool empty() const {
1755     return key.empty();
1756   }
1757
1758   void set_key(const rgw_obj_key& k) {
1759     key = k;
1760   }
1761
1762   string get_oid() const {
1763     return key.get_oid();
1764   }
1765
1766   const string& get_hash_object() const {
1767     return index_hash_source.empty() ? key.name : index_hash_source;
1768   }
1769
1770   void set_in_extra_data(bool val) {
1771     in_extra_data = val;
1772   }
1773
1774   bool is_in_extra_data() const {
1775     return in_extra_data;
1776   }
1777
1778   void encode(bufferlist& bl) const {
1779     ENCODE_START(6, 6, bl);
1780     encode(bucket, bl);
1781     encode(key.ns, bl);
1782     encode(key.name, bl);
1783     encode(key.instance, bl);
1784 //    encode(placement_id, bl);
1785     ENCODE_FINISH(bl);
1786   }
1787   void decode(bufferlist::const_iterator& bl) {
1788     DECODE_START_LEGACY_COMPAT_LEN(6, 3, 3, bl);
1789     if (struct_v < 6) {
1790       string s;
1791       decode(bucket.name, bl); /* bucket.name */
1792       decode(s, bl); /* loc */
1793       decode(key.ns, bl);
1794       decode(key.name, bl);
1795       if (struct_v >= 2)
1796         decode(bucket, bl);
1797       if (struct_v >= 4)
1798         decode(key.instance, bl);
1799       if (key.ns.empty() && key.instance.empty()) {
1800         if (key.name[0] == '_') {
1801           key.name = key.name.substr(1);
1802         }
1803       } else {
1804         if (struct_v >= 5) {
1805           decode(key.name, bl);
1806         } else {
1807           ssize_t pos = key.name.find('_', 1);
1808           if (pos < 0) {
1809             throw buffer::malformed_input();
1810           }
1811           key.name = key.name.substr(pos + 1);
1812         }
1813       }
1814     } else {
1815       decode(bucket, bl);
1816       decode(key.ns, bl);
1817       decode(key.name, bl);
1818       decode(key.instance, bl);
1819 //      decode(placement_id, bl);
1820     }
1821     DECODE_FINISH(bl);
1822   }
1823   void dump(Formatter *f) const;
1824   static void generate_test_instances(list<rgw_obj*>& o);
1825
1826   bool operator==(const rgw_obj& o) const {
1827     return (key == o.key) &&
1828            (bucket == o.bucket);
1829   }
1830   bool operator<(const rgw_obj& o) const {
1831     int r = key.name.compare(o.key.name);
1832     if (r == 0) {
1833       r = bucket.bucket_id.compare(o.bucket.bucket_id); /* not comparing bucket.name, if bucket_id is equal so will be bucket.name */
1834       if (r == 0) {
1835         r = key.ns.compare(o.key.ns);
1836         if (r == 0) {
1837           r = key.instance.compare(o.key.instance);
1838         }
1839       }
1840     }
1841
1842     return (r < 0);
1843   }
1844
1845   const rgw_pool& get_explicit_data_pool() {
1846     if (!in_extra_data || bucket.explicit_placement.data_extra_pool.empty()) {
1847       return bucket.explicit_placement.data_pool;
1848     }
1849     return bucket.explicit_placement.data_extra_pool;
1850   }
1851 };
1852 WRITE_CLASS_ENCODER(rgw_obj)
1853
1854 struct rgw_cache_entry_info {
1855   string cache_locator;
1856   uint64_t gen;
1857
1858   rgw_cache_entry_info() : gen(0) {}
1859 };
1860
1861 inline ostream& operator<<(ostream& out, const rgw_obj &o) {
1862   return out << o.bucket.name << ":" << o.get_oid();
1863 }
1864
1865 static inline void buf_to_hex(const unsigned char* const buf,
1866                               const size_t len,
1867                               char* const str)
1868 {
1869   str[0] = '\0';
1870   for (size_t i = 0; i < len; i++) {
1871     ::sprintf(&str[i*2], "%02x", static_cast<int>(buf[i]));
1872   }
1873 }
1874
1875 template<size_t N> static inline std::array<char, N * 2 + 1>
1876 buf_to_hex(const std::array<unsigned char, N>& buf)
1877 {
1878   static_assert(N > 0, "The input array must be at least one element long");
1879
1880   std::array<char, N * 2 + 1> hex_dest;
1881   buf_to_hex(buf.data(), N, hex_dest.data());
1882   return hex_dest;
1883 }
1884
1885 static inline int hexdigit(char c)
1886 {
1887   if (c >= '0' && c <= '9')
1888     return (c - '0');
1889   c = toupper(c);
1890   if (c >= 'A' && c <= 'F')
1891     return c - 'A' + 0xa;
1892   return -EINVAL;
1893 }
1894
1895 static inline int hex_to_buf(const char *hex, char *buf, int len)
1896 {
1897   int i = 0;
1898   const char *p = hex;
1899   while (*p) {
1900     if (i >= len)
1901       return -EINVAL;
1902     buf[i] = 0;
1903     int d = hexdigit(*p);
1904     if (d < 0)
1905       return d;
1906     buf[i] = d << 4;
1907     p++;
1908     if (!*p)
1909       return -EINVAL;
1910     d = hexdigit(*p);
1911     if (d < 0)
1912       return d;
1913     buf[i] += d;
1914     i++;
1915     p++;
1916   }
1917   return i;
1918 }
1919
1920 static inline int rgw_str_to_bool(const char *s, int def_val)
1921 {
1922   if (!s)
1923     return def_val;
1924
1925   return (strcasecmp(s, "true") == 0 ||
1926           strcasecmp(s, "on") == 0 ||
1927           strcasecmp(s, "yes") == 0 ||
1928           strcasecmp(s, "1") == 0);
1929 }
1930
1931 static inline void append_rand_alpha(CephContext *cct, const string& src, string& dest, int len)
1932 {
1933   dest = src;
1934   char buf[len + 1];
1935   gen_rand_alphanumeric(cct, buf, len);
1936   dest.append("_");
1937   dest.append(buf);
1938 }
1939
1940 static inline const char *rgw_obj_category_name(RGWObjCategory category)
1941 {
1942   switch (category) {
1943   case RGWObjCategory::None:
1944     return "rgw.none";
1945   case RGWObjCategory::Main:
1946     return "rgw.main";
1947   case RGWObjCategory::Shadow:
1948     return "rgw.shadow";
1949   case RGWObjCategory::MultiMeta:
1950     return "rgw.multimeta";
1951   }
1952
1953   return "unknown";
1954 }
1955
1956 static inline uint64_t rgw_rounded_kb(uint64_t bytes)
1957 {
1958   return (bytes + 1023) / 1024;
1959 }
1960
1961 static inline uint64_t rgw_rounded_objsize(uint64_t bytes)
1962 {
1963   return ((bytes + 4095) & ~4095);
1964 }
1965
1966 static inline uint64_t rgw_rounded_objsize_kb(uint64_t bytes)
1967 {
1968   return ((bytes + 4095) & ~4095) / 1024;
1969 }
1970
1971 /* implement combining step, S3 header canonicalization;  k is a
1972  * valid header and in lc form */
1973 void rgw_add_amz_meta_header(
1974   meta_map_t& x_meta_map,
1975   const std::string& k,
1976   const std::string& v);
1977
1978 extern string rgw_string_unquote(const string& s);
1979 extern void parse_csv_string(const string& ival, vector<string>& ovals);
1980 extern int parse_key_value(string& in_str, string& key, string& val);
1981 extern int parse_key_value(string& in_str, const char *delim, string& key, string& val);
1982
1983 extern boost::optional<std::pair<std::string_view, std::string_view>>
1984 parse_key_value(const std::string_view& in_str,
1985                 const std::string_view& delim);
1986 extern boost::optional<std::pair<std::string_view, std::string_view>>
1987 parse_key_value(const std::string_view& in_str);
1988
1989
1990 /** time parsing */
1991 extern int parse_time(const char *time_str, real_time *time);
1992 extern bool parse_rfc2616(const char *s, struct tm *t);
1993 extern bool parse_iso8601(const char *s, struct tm *t, uint32_t *pns = NULL, bool extended_format = true);
1994 extern string rgw_trim_whitespace(const string& src);
1995 extern std::string_view rgw_trim_whitespace(const std::string_view& src);
1996 extern string rgw_trim_quotes(const string& val);
1997
1998 extern void rgw_to_iso8601(const real_time& t, char *dest, int buf_size);
1999 extern void rgw_to_iso8601(const real_time& t, string *dest);
2000 extern std::string rgw_to_asctime(const utime_t& t);
2001
2002 struct perm_state_base {
2003   CephContext *cct;
2004   const rgw::IAM::Environment& env;
2005   rgw::auth::Identity *identity;
2006   const RGWBucketInfo bucket_info;
2007   int perm_mask;
2008   bool defer_to_bucket_acls;
2009   boost::optional<PublicAccessBlockConfiguration> bucket_access_conf;
2010
2011   perm_state_base(CephContext *_cct,
2012                   const rgw::IAM::Environment& _env,
2013                   rgw::auth::Identity *_identity,
2014                   const RGWBucketInfo& _bucket_info,
2015                   int _perm_mask,
2016                   bool _defer_to_bucket_acls,
2017                   boost::optional<PublicAccessBlockConfiguration> _bucket_acess_conf = boost::none) :
2018                                                 cct(_cct),
2019                                                 env(_env),
2020                                                 identity(_identity),
2021                                                 bucket_info(_bucket_info),
2022                                                 perm_mask(_perm_mask),
2023                                                 defer_to_bucket_acls(_defer_to_bucket_acls),
2024                                                 bucket_access_conf(_bucket_acess_conf)
2025   {}
2026
2027   virtual ~perm_state_base() {}
2028
2029   virtual const char *get_referer() const = 0;
2030   virtual std::optional<bool> get_request_payer() const = 0; /*
2031                                                               * empty state means that request_payer param was not passed in
2032                                                               */
2033
2034 };
2035
2036 struct perm_state : public perm_state_base {
2037   const char *referer;
2038   bool request_payer;
2039
2040   perm_state(CephContext *_cct,
2041              const rgw::IAM::Environment& _env,
2042              rgw::auth::Identity *_identity,
2043              const RGWBucketInfo& _bucket_info,
2044              int _perm_mask,
2045              bool _defer_to_bucket_acls,
2046              const char *_referer,
2047              bool _request_payer) : perm_state_base(_cct,
2048                                                     _env,
2049                                                     _identity,
2050                                                     _bucket_info,
2051                                                     _perm_mask,
2052                                                     _defer_to_bucket_acls),
2053                                     referer(_referer),
2054                                     request_payer(_request_payer) {}
2055
2056   const char *get_referer() const override {
2057     return referer;
2058   }
2059
2060   std::optional<bool> get_request_payer() const override {
2061     return request_payer;
2062   }
2063 };
2064
2065 /** Check if the req_state's user has the necessary permissions
2066  * to do the requested action  */
2067 bool verify_bucket_permission_no_policy(
2068   const DoutPrefixProvider* dpp,
2069   struct perm_state_base * const s,
2070   RGWAccessControlPolicy * const user_acl,
2071   RGWAccessControlPolicy * const bucket_acl,
2072   const int perm);
2073
2074 bool verify_user_permission_no_policy(const DoutPrefixProvider* dpp,
2075                                       struct perm_state_base * const s,
2076                                       RGWAccessControlPolicy * const user_acl,
2077                                       const int perm);
2078
2079 bool verify_object_permission_no_policy(const DoutPrefixProvider* dpp,
2080                                         struct perm_state_base * const s,
2081                                         RGWAccessControlPolicy * const user_acl,
2082                                         RGWAccessControlPolicy * const bucket_acl,
2083                                         RGWAccessControlPolicy * const object_acl,
2084                                         const int perm);
2085
2086 /** Check if the req_state's user has the necessary permissions
2087  * to do the requested action */
2088 rgw::IAM::Effect eval_user_policies(const vector<rgw::IAM::Policy>& user_policies,
2089                           const rgw::IAM::Environment& env,
2090                           boost::optional<const rgw::auth::Identity&> id,
2091                           const uint64_t op,
2092                           const rgw::ARN& arn);
2093 bool verify_user_permission(const DoutPrefixProvider* dpp,
2094                             struct req_state * const s,
2095                             RGWAccessControlPolicy * const user_acl,
2096                             const vector<rgw::IAM::Policy>& user_policies,
2097                             const rgw::ARN& res,
2098                             const uint64_t op);
2099 bool verify_user_permission_no_policy(const DoutPrefixProvider* dpp,
2100                                       struct req_state * const s,
2101                                       RGWAccessControlPolicy * const user_acl,
2102                                       const int perm);
2103 bool verify_user_permission(const DoutPrefixProvider* dpp,
2104                             struct req_state * const s,
2105                             const rgw::ARN& res,
2106                             const uint64_t op);
2107 bool verify_user_permission_no_policy(const DoutPrefixProvider* dpp,
2108                                       struct req_state * const s,
2109                                       int perm);
2110 bool verify_bucket_permission(
2111   const DoutPrefixProvider* dpp,
2112   struct req_state * const s,
2113   const rgw_bucket& bucket,
2114   RGWAccessControlPolicy * const user_acl,
2115   RGWAccessControlPolicy * const bucket_acl,
2116   const boost::optional<rgw::IAM::Policy>& bucket_policy,
2117   const vector<rgw::IAM::Policy>& user_policies,
2118   const uint64_t op);
2119 bool verify_bucket_permission(const DoutPrefixProvider* dpp, struct req_state * const s, const uint64_t op);
2120 bool verify_bucket_permission_no_policy(
2121   const DoutPrefixProvider* dpp,
2122   struct req_state * const s,
2123   RGWAccessControlPolicy * const user_acl,
2124   RGWAccessControlPolicy * const bucket_acl,
2125   const int perm);
2126 bool verify_bucket_permission_no_policy(const DoutPrefixProvider* dpp,
2127                                         struct req_state * const s,
2128                                         const int perm);
2129 int verify_bucket_owner_or_policy(struct req_state* const s,
2130                                   const uint64_t op);
2131 extern bool verify_object_permission(
2132   const DoutPrefixProvider* dpp,
2133   struct req_state * const s,
2134   const rgw_obj& obj,
2135   RGWAccessControlPolicy * const user_acl,
2136   RGWAccessControlPolicy * const bucket_acl,
2137   RGWAccessControlPolicy * const object_acl,
2138   const boost::optional<rgw::IAM::Policy>& bucket_policy,
2139   const vector<rgw::IAM::Policy>& user_policies,
2140   const uint64_t op);
2141 extern bool verify_object_permission(const DoutPrefixProvider* dpp, struct req_state *s, uint64_t op);
2142 extern bool verify_object_permission_no_policy(
2143   const DoutPrefixProvider* dpp,
2144   struct req_state * const s,
2145   RGWAccessControlPolicy * const user_acl,
2146   RGWAccessControlPolicy * const bucket_acl,
2147   RGWAccessControlPolicy * const object_acl,
2148   int perm);
2149 extern bool verify_object_permission_no_policy(const DoutPrefixProvider* dpp, struct req_state *s,
2150                                                int perm);
2151 extern int verify_object_lock(
2152   const DoutPrefixProvider* dpp,
2153   const rgw::sal::RGWAttrs& attrs,
2154   const bool bypass_perm,
2155   const bool bypass_governance_mode);
2156
2157 /** Convert an input URL into a sane object name
2158  * by converting %-escaped strings into characters, etc*/
2159 extern void rgw_uri_escape_char(char c, string& dst);
2160 extern std::string url_decode(const std::string_view& src_str,
2161                               bool in_query = false);
2162 extern void url_encode(const std::string& src, string& dst,
2163                        bool encode_slash = true);
2164 extern std::string url_encode(const std::string& src, bool encode_slash = true);
2165 extern std::string url_remove_prefix(const std::string& url); // Removes hhtp, https and www from url
2166 /* destination should be CEPH_CRYPTO_HMACSHA1_DIGESTSIZE bytes long */
2167 extern void calc_hmac_sha1(const char *key, int key_len,
2168                           const char *msg, int msg_len, char *dest);
2169
2170 static inline sha1_digest_t
2171 calc_hmac_sha1(const std::string_view& key, const std::string_view& msg) {
2172   sha1_digest_t dest;
2173   calc_hmac_sha1(key.data(), key.size(), msg.data(), msg.size(),
2174                  reinterpret_cast<char*>(dest.v));
2175   return dest;
2176 }
2177
2178 /* destination should be CEPH_CRYPTO_HMACSHA256_DIGESTSIZE bytes long */
2179 extern void calc_hmac_sha256(const char *key, int key_len,
2180                              const char *msg, int msg_len,
2181                              char *dest);
2182
2183 static inline sha256_digest_t
2184 calc_hmac_sha256(const char *key, const int key_len,
2185                  const char *msg, const int msg_len) {
2186   sha256_digest_t dest;
2187   calc_hmac_sha256(key, key_len, msg, msg_len,
2188                    reinterpret_cast<char*>(dest.v));
2189   return dest;
2190 }
2191
2192 static inline sha256_digest_t
2193 calc_hmac_sha256(const std::string_view& key, const std::string_view& msg) {
2194   sha256_digest_t dest;
2195   calc_hmac_sha256(key.data(), key.size(),
2196                    msg.data(), msg.size(),
2197                    reinterpret_cast<char*>(dest.v));
2198   return dest;
2199 }
2200
2201 static inline sha256_digest_t
2202 calc_hmac_sha256(const sha256_digest_t &key,
2203                  const std::string_view& msg) {
2204   sha256_digest_t dest;
2205   calc_hmac_sha256(reinterpret_cast<const char*>(key.v), sha256_digest_t::SIZE,
2206                    msg.data(), msg.size(),
2207                    reinterpret_cast<char*>(dest.v));
2208   return dest;
2209 }
2210
2211 static inline sha256_digest_t
2212 calc_hmac_sha256(const std::vector<unsigned char>& key,
2213                  const std::string_view& msg) {
2214   sha256_digest_t dest;
2215   calc_hmac_sha256(reinterpret_cast<const char*>(key.data()), key.size(),
2216                    msg.data(), msg.size(),
2217                    reinterpret_cast<char*>(dest.v));
2218   return dest;
2219 }
2220
2221 template<size_t KeyLenN>
2222 static inline sha256_digest_t
2223 calc_hmac_sha256(const std::array<unsigned char, KeyLenN>& key,
2224                  const std::string_view& msg) {
2225   sha256_digest_t dest;
2226   calc_hmac_sha256(reinterpret_cast<const char*>(key.data()), key.size(),
2227                    msg.data(), msg.size(),
2228                    reinterpret_cast<char*>(dest.v));
2229   return dest;
2230 }
2231
2232 extern sha256_digest_t calc_hash_sha256(const std::string_view& msg);
2233
2234 extern ceph::crypto::SHA256* calc_hash_sha256_open_stream();
2235 extern void calc_hash_sha256_update_stream(ceph::crypto::SHA256* hash,
2236                                            const char* msg,
2237                                            int len);
2238 extern std::string calc_hash_sha256_close_stream(ceph::crypto::SHA256** phash);
2239 extern std::string calc_hash_sha256_restart_stream(ceph::crypto::SHA256** phash);
2240
2241 extern int rgw_parse_op_type_list(const string& str, uint32_t *perm);
2242
2243 static constexpr uint32_t MATCH_POLICY_ACTION = 0x01;
2244 static constexpr uint32_t MATCH_POLICY_RESOURCE = 0x02;
2245 static constexpr uint32_t MATCH_POLICY_ARN = 0x04;
2246 static constexpr uint32_t MATCH_POLICY_STRING = 0x08;
2247
2248 extern bool match_policy(std::string_view pattern, std::string_view input,
2249                          uint32_t flag);
2250
2251 extern string camelcase_dash_http_attr(const string& orig);
2252 extern string lowercase_dash_http_attr(const string& orig);
2253
2254 void rgw_setup_saved_curl_handles();
2255 void rgw_release_all_curl_handles();
2256
2257 static inline void rgw_escape_str(const string& s, char esc_char,
2258                                   char special_char, string *dest)
2259 {
2260   const char *src = s.c_str();
2261   char dest_buf[s.size() * 2 + 1];
2262   char *destp = dest_buf;
2263
2264   for (size_t i = 0; i < s.size(); i++) {
2265     char c = src[i];
2266     if (c == esc_char || c == special_char) {
2267       *destp++ = esc_char;
2268     }
2269     *destp++ = c;
2270   }
2271   *destp++ = '\0';
2272   *dest = dest_buf;
2273 }
2274
2275 static inline ssize_t rgw_unescape_str(const string& s, ssize_t ofs,
2276                                        char esc_char, char special_char,
2277                                        string *dest)
2278 {
2279   const char *src = s.c_str();
2280   char dest_buf[s.size() + 1];
2281   char *destp = dest_buf;
2282   bool esc = false;
2283
2284   dest_buf[0] = '\0';
2285
2286   for (size_t i = ofs; i < s.size(); i++) {
2287     char c = src[i];
2288     if (!esc && c == esc_char) {
2289       esc = true;
2290       continue;
2291     }
2292     if (!esc && c == special_char) {
2293       *destp = '\0';
2294       *dest = dest_buf;
2295       return (ssize_t)i + 1;
2296     }
2297     *destp++ = c;
2298     esc = false;
2299   }
2300   *destp = '\0';
2301   *dest = dest_buf;
2302   return string::npos;
2303 }
2304
2305 static inline string rgw_bl_str(ceph::buffer::list& raw)
2306 {
2307   size_t len = raw.length();
2308   string s(raw.c_str(), len);
2309   while (len && !s[len - 1]) {
2310     --len;
2311     s.resize(len);
2312   }
2313   return s;
2314 }
2315
2316 template <typename T>
2317 int decode_bl(bufferlist& bl, T& t)
2318 {
2319   auto iter = bl.cbegin();
2320   try {
2321     decode(t, iter);
2322   } catch (buffer::error& err) {
2323     return -EIO;
2324   }
2325   return 0;
2326 }
2327
2328 #endif