ceph/src/rgw/rgw_common.h

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab ft=cpp
   3
   4 /*
   5  * Ceph - scalable distributed file system
   6  *
   7  * Copyright (C) 2004-2009 Sage Weil <sage@newdream.net>
   8  * Copyright (C) 2015 Yehuda Sadeh <yehuda@redhat.com>
   9  *
  10  * This is free software; you can redistribute it and/or
  11  * modify it under the terms of the GNU Lesser General Public
  12  * License version 2.1, as published by the Free Software
  13  * Foundation.  See file COPYING.
  14  *
  15  */
  16
  17 #pragma once
  18
  19 #include <array>
  20 #include <string_view>
  21 #include <atomic>
  22 #include <unordered_map>
  23
  24 #include "common/ceph_crypto.h"
  25 #include "common/random_string.h"
  26 #include "rgw_acl.h"
  27 #include "rgw_bucket_layout.h"
  28 #include "rgw_cors.h"
  29 #include "rgw_iam_policy.h"
  30 #include "rgw_quota.h"
  31 #include "rgw_string.h"
  32 #include "common/async/yield_context.h"
  33 #include "rgw_website.h"
  34 #include "rgw_object_lock.h"
  35 #include "rgw_tag.h"
  36 #include "rgw_op_type.h"
  37 #include "rgw_sync_policy.h"
  38 #include "cls/version/cls_version_types.h"
  39 #include "cls/user/cls_user_types.h"
  40 #include "cls/rgw/cls_rgw_types.h"
  41 #include "include/rados/librados.hpp"
  42 #include "rgw_public_access.h"
  43 #include "common/tracer.h"
  44
  45 namespace ceph {
  46   class Formatter;
  47 }
  48
  49 namespace rgw::sal {
  50   class User;
  51   class Bucket;
  52   class Object;
  53   using Attrs = std::map<std::string, ceph::buffer::list>;
  54 }
  55
  56 using ceph::crypto::MD5;
  57
  58 #define RGW_ATTR_PREFIX  "user.rgw."
  59
  60 #define RGW_HTTP_RGWX_ATTR_PREFIX "RGWX_ATTR_"
  61 #define RGW_HTTP_RGWX_ATTR_PREFIX_OUT "Rgwx-Attr-"
  62
  63 #define RGW_AMZ_PREFIX "x-amz-"
  64 #define RGW_AMZ_META_PREFIX RGW_AMZ_PREFIX "meta-"
  65 #define RGW_AMZ_WEBSITE_REDIRECT_LOCATION RGW_AMZ_PREFIX "website-redirect-location"
  66 #define RGW_AMZ_TAG_COUNT RGW_AMZ_PREFIX "tagging-count"
  67
  68 #define RGW_SYS_PARAM_PREFIX "rgwx-"
  69
  70 #define RGW_ATTR_ACL            RGW_ATTR_PREFIX "acl"
  71 #define RGW_ATTR_RATELIMIT              RGW_ATTR_PREFIX "ratelimit"
  72 #define RGW_ATTR_LC            RGW_ATTR_PREFIX "lc"
  73 #define RGW_ATTR_CORS           RGW_ATTR_PREFIX "cors"
  74 #define RGW_ATTR_ETAG           RGW_ATTR_PREFIX "etag"
  75 #define RGW_ATTR_BUCKETS        RGW_ATTR_PREFIX "buckets"
  76 #define RGW_ATTR_META_PREFIX    RGW_ATTR_PREFIX RGW_AMZ_META_PREFIX
  77 #define RGW_ATTR_CONTENT_TYPE   RGW_ATTR_PREFIX "content_type"
  78 #define RGW_ATTR_CACHE_CONTROL  RGW_ATTR_PREFIX "cache_control"
  79 #define RGW_ATTR_CONTENT_DISP   RGW_ATTR_PREFIX "content_disposition"
  80 #define RGW_ATTR_CONTENT_ENC    RGW_ATTR_PREFIX "content_encoding"
  81 #define RGW_ATTR_CONTENT_LANG   RGW_ATTR_PREFIX "content_language"
  82 #define RGW_ATTR_EXPIRES        RGW_ATTR_PREFIX "expires"
  83 #define RGW_ATTR_DELETE_AT      RGW_ATTR_PREFIX "delete_at"
  84 #define RGW_ATTR_ID_TAG         RGW_ATTR_PREFIX "idtag"
  85 #define RGW_ATTR_TAIL_TAG       RGW_ATTR_PREFIX "tail_tag"
  86 #define RGW_ATTR_SHADOW_OBJ     RGW_ATTR_PREFIX "shadow_name"
  87 #define RGW_ATTR_MANIFEST       RGW_ATTR_PREFIX "manifest"
  88 #define RGW_ATTR_USER_MANIFEST  RGW_ATTR_PREFIX "user_manifest"
  89 #define RGW_ATTR_AMZ_WEBSITE_REDIRECT_LOCATION  RGW_ATTR_PREFIX RGW_AMZ_WEBSITE_REDIRECT_LOCATION
  90 #define RGW_ATTR_SLO_MANIFEST   RGW_ATTR_PREFIX "slo_manifest"
  91 /* Information whether an object is SLO or not must be exposed to
  92  * user through custom HTTP header named X-Static-Large-Object. */
  93 #define RGW_ATTR_SLO_UINDICATOR RGW_ATTR_META_PREFIX "static-large-object"
  94 #define RGW_ATTR_X_ROBOTS_TAG   RGW_ATTR_PREFIX "x-robots-tag"
  95 #define RGW_ATTR_STORAGE_CLASS  RGW_ATTR_PREFIX "storage_class"
  96
  97 /* S3 Object Lock*/
  98 #define RGW_ATTR_OBJECT_LOCK        RGW_ATTR_PREFIX "object-lock"
  99 #define RGW_ATTR_OBJECT_RETENTION   RGW_ATTR_PREFIX "object-retention"
 100 #define RGW_ATTR_OBJECT_LEGAL_HOLD  RGW_ATTR_PREFIX "object-legal-hold"
 101
 102
 103 #define RGW_ATTR_PG_VER         RGW_ATTR_PREFIX "pg_ver"
 104 #define RGW_ATTR_SOURCE_ZONE    RGW_ATTR_PREFIX "source_zone"
 105 #define RGW_ATTR_TAGS           RGW_ATTR_PREFIX RGW_AMZ_PREFIX "tagging"
 106
 107 #define RGW_ATTR_TEMPURL_KEY1   RGW_ATTR_META_PREFIX "temp-url-key"
 108 #define RGW_ATTR_TEMPURL_KEY2   RGW_ATTR_META_PREFIX "temp-url-key-2"
 109
 110 /* Account/container quota of the Swift API. */
 111 #define RGW_ATTR_QUOTA_NOBJS    RGW_ATTR_META_PREFIX "quota-count"
 112 #define RGW_ATTR_QUOTA_MSIZE    RGW_ATTR_META_PREFIX "quota-bytes"
 113
 114 /* Static Web Site of Swift API. */
 115 #define RGW_ATTR_WEB_INDEX      RGW_ATTR_META_PREFIX "web-index"
 116 #define RGW_ATTR_WEB_ERROR      RGW_ATTR_META_PREFIX "web-error"
 117 #define RGW_ATTR_WEB_LISTINGS   RGW_ATTR_META_PREFIX "web-listings"
 118 #define RGW_ATTR_WEB_LIST_CSS   RGW_ATTR_META_PREFIX "web-listings-css"
 119 #define RGW_ATTR_SUBDIR_MARKER  RGW_ATTR_META_PREFIX "web-directory-type"
 120
 121 #define RGW_ATTR_OLH_PREFIX     RGW_ATTR_PREFIX "olh."
 122
 123 #define RGW_ATTR_OLH_INFO       RGW_ATTR_OLH_PREFIX "info"
 124 #define RGW_ATTR_OLH_VER        RGW_ATTR_OLH_PREFIX "ver"
 125 #define RGW_ATTR_OLH_ID_TAG     RGW_ATTR_OLH_PREFIX "idtag"
 126 #define RGW_ATTR_OLH_PENDING_PREFIX RGW_ATTR_OLH_PREFIX "pending."
 127
 128 #define RGW_ATTR_COMPRESSION    RGW_ATTR_PREFIX "compression"
 129
 130 #define RGW_ATTR_APPEND_PART_NUM    RGW_ATTR_PREFIX "append_part_num"
 131
 132 /* IAM Policy */
 133 #define RGW_ATTR_IAM_POLICY     RGW_ATTR_PREFIX "iam-policy"
 134 #define RGW_ATTR_USER_POLICY    RGW_ATTR_PREFIX "user-policy"
 135 #define RGW_ATTR_PUBLIC_ACCESS  RGW_ATTR_PREFIX "public-access"
 136
 137 /* RGW File Attributes */
 138 #define RGW_ATTR_UNIX_KEY1      RGW_ATTR_PREFIX "unix-key1"
 139 #define RGW_ATTR_UNIX1          RGW_ATTR_PREFIX "unix1"
 140
 141 #define RGW_ATTR_CRYPT_PREFIX   RGW_ATTR_PREFIX "crypt."
 142 #define RGW_ATTR_CRYPT_MODE     RGW_ATTR_CRYPT_PREFIX "mode"
 143 #define RGW_ATTR_CRYPT_KEYMD5   RGW_ATTR_CRYPT_PREFIX "keymd5"
 144 #define RGW_ATTR_CRYPT_KEYID    RGW_ATTR_CRYPT_PREFIX "keyid"
 145 #define RGW_ATTR_CRYPT_KEYSEL   RGW_ATTR_CRYPT_PREFIX "keysel"
 146 #define RGW_ATTR_CRYPT_CONTEXT  RGW_ATTR_CRYPT_PREFIX "context"
 147 #define RGW_ATTR_CRYPT_DATAKEY  RGW_ATTR_CRYPT_PREFIX "datakey"
 148
 149 /* SSE-S3 Encryption Attributes */
 150 #define RGW_ATTR_BUCKET_ENCRYPTION_PREFIX RGW_ATTR_PREFIX "sse-s3."
 151 #define RGW_ATTR_BUCKET_ENCRYPTION_POLICY RGW_ATTR_BUCKET_ENCRYPTION_PREFIX "policy"
 152 #define RGW_ATTR_BUCKET_ENCRYPTION_KEY_ID RGW_ATTR_BUCKET_ENCRYPTION_PREFIX "key-id"
 153
 154 #define RGW_ATTR_TRACE RGW_ATTR_PREFIX "trace"
 155
 156 #define RGW_FORMAT_PLAIN        0
 157 #define RGW_FORMAT_XML          1
 158 #define RGW_FORMAT_JSON         2
 159 #define RGW_FORMAT_HTML         3
 160
 161 #define RGW_CAP_READ            0x1
 162 #define RGW_CAP_WRITE           0x2
 163 #define RGW_CAP_ALL             (RGW_CAP_READ | RGW_CAP_WRITE)
 164
 165 #define RGW_REST_SWIFT          0x1
 166 #define RGW_REST_SWIFT_AUTH     0x2
 167 #define RGW_REST_S3             0x4
 168 #define RGW_REST_WEBSITE     0x8
 169 #define RGW_REST_STS            0x10
 170 #define RGW_REST_IAM            0x20
 171
 172 #define RGW_SUSPENDED_USER_AUID (uint64_t)-2
 173
 174 #define RGW_OP_TYPE_READ         0x01
 175 #define RGW_OP_TYPE_WRITE        0x02
 176 #define RGW_OP_TYPE_DELETE       0x04
 177
 178 #define RGW_OP_TYPE_MODIFY       (RGW_OP_TYPE_WRITE | RGW_OP_TYPE_DELETE)
 179 #define RGW_OP_TYPE_ALL          (RGW_OP_TYPE_READ | RGW_OP_TYPE_WRITE | RGW_OP_TYPE_DELETE)
 180
 181 #define RGW_DEFAULT_MAX_BUCKETS 1000
 182
 183 #define RGW_DEFER_TO_BUCKET_ACLS_RECURSE 1
 184 #define RGW_DEFER_TO_BUCKET_ACLS_FULL_CONTROL 2
 185
 186 #define STATUS_CREATED           1900
 187 #define STATUS_ACCEPTED          1901
 188 #define STATUS_NO_CONTENT        1902
 189 #define STATUS_PARTIAL_CONTENT   1903
 190 #define STATUS_REDIRECT          1904
 191 #define STATUS_NO_APPLY          1905
 192 #define STATUS_APPLIED           1906
 193
 194 #define ERR_INVALID_BUCKET_NAME  2000
 195 #define ERR_INVALID_OBJECT_NAME  2001
 196 #define ERR_NO_SUCH_BUCKET       2002
 197 #define ERR_METHOD_NOT_ALLOWED   2003
 198 #define ERR_INVALID_DIGEST       2004
 199 #define ERR_BAD_DIGEST           2005
 200 #define ERR_UNRESOLVABLE_EMAIL   2006
 201 #define ERR_INVALID_PART         2007
 202 #define ERR_INVALID_PART_ORDER   2008
 203 #define ERR_NO_SUCH_UPLOAD       2009
 204 #define ERR_REQUEST_TIMEOUT      2010
 205 #define ERR_LENGTH_REQUIRED      2011
 206 #define ERR_REQUEST_TIME_SKEWED  2012
 207 #define ERR_BUCKET_EXISTS        2013
 208 #define ERR_BAD_URL              2014
 209 #define ERR_PRECONDITION_FAILED  2015
 210 #define ERR_NOT_MODIFIED         2016
 211 #define ERR_INVALID_UTF8         2017
 212 #define ERR_UNPROCESSABLE_ENTITY 2018
 213 #define ERR_TOO_LARGE            2019
 214 #define ERR_TOO_MANY_BUCKETS     2020
 215 #define ERR_INVALID_REQUEST      2021
 216 #define ERR_TOO_SMALL            2022
 217 #define ERR_NOT_FOUND            2023
 218 #define ERR_PERMANENT_REDIRECT   2024
 219 #define ERR_LOCKED               2025
 220 #define ERR_QUOTA_EXCEEDED       2026
 221 #define ERR_SIGNATURE_NO_MATCH   2027
 222 #define ERR_INVALID_ACCESS_KEY   2028
 223 #define ERR_MALFORMED_XML        2029
 224 #define ERR_USER_EXIST           2030
 225 #define ERR_NOT_SLO_MANIFEST     2031
 226 #define ERR_EMAIL_EXIST          2032
 227 #define ERR_KEY_EXIST            2033
 228 #define ERR_INVALID_SECRET_KEY   2034
 229 #define ERR_INVALID_KEY_TYPE     2035
 230 #define ERR_INVALID_CAP          2036
 231 #define ERR_INVALID_TENANT_NAME  2037
 232 #define ERR_WEBSITE_REDIRECT     2038
 233 #define ERR_NO_SUCH_WEBSITE_CONFIGURATION 2039
 234 #define ERR_AMZ_CONTENT_SHA256_MISMATCH 2040
 235 #define ERR_NO_SUCH_LC           2041
 236 #define ERR_NO_SUCH_USER         2042
 237 #define ERR_NO_SUCH_SUBUSER      2043
 238 #define ERR_MFA_REQUIRED         2044
 239 #define ERR_NO_SUCH_CORS_CONFIGURATION 2045
 240 #define ERR_NO_SUCH_OBJECT_LOCK_CONFIGURATION  2046
 241 #define ERR_INVALID_RETENTION_PERIOD 2047
 242 #define ERR_NO_SUCH_BUCKET_ENCRYPTION_CONFIGURATION 2048
 243 #define ERR_USER_SUSPENDED       2100
 244 #define ERR_INTERNAL_ERROR       2200
 245 #define ERR_NOT_IMPLEMENTED      2201
 246 #define ERR_SERVICE_UNAVAILABLE  2202
 247 #define ERR_ROLE_EXISTS          2203
 248 #define ERR_MALFORMED_DOC        2204
 249 #define ERR_NO_ROLE_FOUND        2205
 250 #define ERR_DELETE_CONFLICT      2206
 251 #define ERR_NO_SUCH_BUCKET_POLICY  2207
 252 #define ERR_INVALID_LOCATION_CONSTRAINT 2208
 253 #define ERR_TAG_CONFLICT         2209
 254 #define ERR_INVALID_TAG          2210
 255 #define ERR_ZERO_IN_URL          2211
 256 #define ERR_MALFORMED_ACL_ERROR  2212
 257 #define ERR_ZONEGROUP_DEFAULT_PLACEMENT_MISCONFIGURATION 2213
 258 #define ERR_INVALID_ENCRYPTION_ALGORITHM                 2214
 259 #define ERR_INVALID_CORS_RULES_ERROR                     2215
 260 #define ERR_NO_CORS_FOUND        2216
 261 #define ERR_INVALID_WEBSITE_ROUTING_RULES_ERROR          2217
 262 #define ERR_RATE_LIMITED         2218
 263 #define ERR_POSITION_NOT_EQUAL_TO_LENGTH                 2219
 264 #define ERR_OBJECT_NOT_APPENDABLE                        2220
 265 #define ERR_INVALID_BUCKET_STATE                         2221
 266 #define ERR_INVALID_OBJECT_STATE                         2222
 267
 268 #define ERR_BUSY_RESHARDING      2300
 269 #define ERR_NO_SUCH_ENTITY       2301
 270 #define ERR_LIMIT_EXCEEDED       2302
 271
 272 // STS Errors
 273 #define ERR_PACKED_POLICY_TOO_LARGE 2400
 274 #define ERR_INVALID_IDENTITY_TOKEN  2401
 275
 276 #define ERR_NO_SUCH_TAG_SET 2402
 277
 278 #ifndef UINT32_MAX
 279 #define UINT32_MAX (0xffffffffu)
 280 #endif
 281
 282 struct req_state;
 283
 284 typedef void *RGWAccessHandle;
 285
 286 enum RGWIntentEvent {
 287   DEL_OBJ = 0,
 288   DEL_DIR = 1,
 289 };
 290
 291 enum HostStyle {
 292   PathStyle = 0,
 293   VirtualStyle = 1,
 294 };
 295
 296 /** Store error returns for output at a different point in the program */
 297 struct rgw_err {
 298   rgw_err();
 299   void clear();
 300   bool is_clear() const;
 301   bool is_err() const;
 302   friend std::ostream& operator<<(std::ostream& oss, const rgw_err &err);
 303
 304   int http_ret;
 305   int ret;
 306   std::string err_code;
 307   std::string message;
 308 };
 309
 310
 311 /* Helper class used for RGWHTTPArgs parsing */
 312 class NameVal
 313 {
 314    const std::string str;
 315    std::string name;
 316    std::string val;
 317  public:
 318     explicit NameVal(const std::string& nv) : str(nv) {}
 319
 320     int parse();
 321
 322     std::string& get_name() { return name; }
 323     std::string& get_val() { return val; }
 324 };
 325
 326 /** Stores the XML arguments associated with the HTTP request in req_state*/
 327 class RGWHTTPArgs {
 328   std::string str, empty_str;
 329   std::map<std::string, std::string> val_map;
 330   std::map<std::string, std::string> sys_val_map;
 331   std::map<std::string, std::string> sub_resources;
 332   bool has_resp_modifier = false;
 333   bool admin_subresource_added = false;
 334  public:
 335   RGWHTTPArgs() = default;
 336   explicit RGWHTTPArgs(const std::string& s, const DoutPrefixProvider *dpp) {
 337       set(s);
 338       parse(dpp);
 339   }
 340
 341   /** Set the arguments; as received */
 342   void set(const std::string& s) {
 343     has_resp_modifier = false;
 344     val_map.clear();
 345     sub_resources.clear();
 346     str = s;
 347   }
 348   /** parse the received arguments */
 349   int parse(const DoutPrefixProvider *dpp);
 350   void append(const std::string& name, const std::string& val);
 351   /** Get the value for a specific argument parameter */
 352   const std::string& get(const std::string& name, bool *exists = NULL) const;
 353   boost::optional<const std::string&>
 354   get_optional(const std::string& name) const;
 355   int get_bool(const std::string& name, bool *val, bool *exists) const;
 356   int get_bool(const char *name, bool *val, bool *exists) const;
 357   void get_bool(const char *name, bool *val, bool def_val) const;
 358   int get_int(const char *name, int *val, int def_val) const;
 359
 360   /** Get the value for specific system argument parameter */
 361   std::string sys_get(const std::string& name, bool *exists = nullptr) const;
 362
 363   /** see if a parameter is contained in this RGWHTTPArgs */
 364   bool exists(const char *name) const {
 365     return (val_map.find(name) != std::end(val_map));
 366   }
 367   bool sub_resource_exists(const char *name) const {
 368     return (sub_resources.find(name) != std::end(sub_resources));
 369   }
 370   bool exist_obj_excl_sub_resource() const {
 371     const char* const obj_sub_resource[] = {"append", "torrent", "uploadId",
 372                                             "partNumber", "versionId"};
 373     for (unsigned i = 0; i != std::size(obj_sub_resource); i++) {
 374       if (sub_resource_exists(obj_sub_resource[i])) return true;
 375     }
 376     return false;
 377   }
 378
 379   std::map<std::string, std::string>& get_params() {
 380     return val_map;
 381   }
 382   const std::map<std::string, std::string>& get_params() const {
 383     return val_map;
 384   }
 385   std::map<std::string, std::string>& get_sys_params() {
 386     return sys_val_map;
 387   }
 388   const std::map<std::string, std::string>& get_sys_params() const {
 389     return sys_val_map;
 390   }
 391   const std::map<std::string, std::string>& get_sub_resources() const {
 392     return sub_resources;
 393   }
 394   unsigned get_num_params() const {
 395     return val_map.size();
 396   }
 397   bool has_response_modifier() const {
 398     return has_resp_modifier;
 399   }
 400   void set_system() { /* make all system params visible */
 401     std::map<std::string, std::string>::iterator iter;
 402     for (iter = sys_val_map.begin(); iter != sys_val_map.end(); ++iter) {
 403       val_map[iter->first] = iter->second;
 404     }
 405   }
 406   const std::string& get_str() {
 407     return str;
 408   }
 409 }; // RGWHTTPArgs
 410
 411 const char *rgw_conf_get(const std::map<std::string, std::string, ltstr_nocase>& conf_map, const char *name, const char *def_val);
 412 int rgw_conf_get_int(const std::map<std::string, std::string, ltstr_nocase>& conf_map, const char *name, int def_val);
 413 bool rgw_conf_get_bool(const std::map<std::string, std::string, ltstr_nocase>& conf_map, const char *name, bool def_val);
 414
 415 class RGWEnv;
 416
 417 class RGWConf {
 418   friend class RGWEnv;
 419   int enable_ops_log;
 420   int enable_usage_log;
 421   uint8_t defer_to_bucket_acls;
 422   void init(CephContext *cct);
 423 public:
 424   RGWConf()
 425     : enable_ops_log(1),
 426       enable_usage_log(1),
 427       defer_to_bucket_acls(0) {
 428   }
 429 };
 430
 431 class RGWEnv {
 432   std::map<std::string, std::string, ltstr_nocase> env_map;
 433   RGWConf conf;
 434 public:
 435   void init(CephContext *cct);
 436   void init(CephContext *cct, char **envp);
 437   void set(std::string name, std::string val);
 438   const char *get(const char *name, const char *def_val = nullptr) const;
 439   int get_int(const char *name, int def_val = 0) const;
 440   bool get_bool(const char *name, bool def_val = 0);
 441   size_t get_size(const char *name, size_t def_val = 0) const;
 442   bool exists(const char *name) const;
 443   bool exists_prefix(const char *prefix) const;
 444   void remove(const char *name);
 445   const std::map<std::string, std::string, ltstr_nocase>& get_map() const { return env_map; }
 446   int get_enable_ops_log() const {
 447     return conf.enable_ops_log;
 448   }
 449
 450   int get_enable_usage_log() const {
 451     return conf.enable_usage_log;
 452   }
 453
 454   int get_defer_to_bucket_acls() const {
 455     return conf.defer_to_bucket_acls;
 456   }
 457 };
 458
 459 // return true if the connection is secure. this either means that the
 460 // connection arrived via ssl, or was forwarded as https by a trusted proxy
 461 bool rgw_transport_is_secure(CephContext *cct, const RGWEnv& env);
 462
 463 enum http_op {
 464   OP_GET,
 465   OP_PUT,
 466   OP_DELETE,
 467   OP_HEAD,
 468   OP_POST,
 469   OP_COPY,
 470   OP_OPTIONS,
 471   OP_UNKNOWN,
 472 };
 473
 474 class RGWAccessControlPolicy;
 475 class JSONObj;
 476
 477 struct RGWAccessKey {
 478   std::string id; // AccessKey
 479   std::string key; // SecretKey
 480   std::string subuser;
 481
 482   RGWAccessKey() {}
 483   RGWAccessKey(std::string _id, std::string _key)
 484     : id(std::move(_id)), key(std::move(_key)) {}
 485
 486   void encode(bufferlist& bl) const {
 487     ENCODE_START(2, 2, bl);
 488     encode(id, bl);
 489     encode(key, bl);
 490     encode(subuser, bl);
 491     ENCODE_FINISH(bl);
 492   }
 493
 494   void decode(bufferlist::const_iterator& bl) {
 495      DECODE_START_LEGACY_COMPAT_LEN_32(2, 2, 2, bl);
 496      decode(id, bl);
 497      decode(key, bl);
 498      decode(subuser, bl);
 499      DECODE_FINISH(bl);
 500   }
 501   void dump(Formatter *f) const;
 502   void dump_plain(Formatter *f) const;
 503   void dump(Formatter *f, const std::string& user, bool swift) const;
 504   static void generate_test_instances(std::list<RGWAccessKey*>& o);
 505
 506   void decode_json(JSONObj *obj);
 507   void decode_json(JSONObj *obj, bool swift);
 508 };
 509 WRITE_CLASS_ENCODER(RGWAccessKey)
 510
 511 struct RGWSubUser {
 512   std::string name;
 513   uint32_t perm_mask;
 514
 515   RGWSubUser() : perm_mask(0) {}
 516   void encode(bufferlist& bl) const {
 517     ENCODE_START(2, 2, bl);
 518     encode(name, bl);
 519     encode(perm_mask, bl);
 520     ENCODE_FINISH(bl);
 521   }
 522
 523   void decode(bufferlist::const_iterator& bl) {
 524      DECODE_START_LEGACY_COMPAT_LEN_32(2, 2, 2, bl);
 525      decode(name, bl);
 526      decode(perm_mask, bl);
 527      DECODE_FINISH(bl);
 528   }
 529   void dump(Formatter *f) const;
 530   void dump(Formatter *f, const std::string& user) const;
 531   static void generate_test_instances(std::list<RGWSubUser*>& o);
 532
 533   void decode_json(JSONObj *obj);
 534 };
 535 WRITE_CLASS_ENCODER(RGWSubUser)
 536
 537 class RGWUserCaps
 538 {
 539   std::map<std::string, uint32_t> caps;
 540
 541   int get_cap(const std::string& cap, std::string& type, uint32_t *perm);
 542   int add_cap(const std::string& cap);
 543   int remove_cap(const std::string& cap);
 544 public:
 545   static int parse_cap_perm(const std::string& str, uint32_t *perm);
 546   int add_from_string(const std::string& str);
 547   int remove_from_string(const std::string& str);
 548
 549   void encode(bufferlist& bl) const {
 550      ENCODE_START(1, 1, bl);
 551      encode(caps, bl);
 552      ENCODE_FINISH(bl);
 553   }
 554   void decode(bufferlist::const_iterator& bl) {
 555      DECODE_START(1, bl);
 556      decode(caps, bl);
 557      DECODE_FINISH(bl);
 558   }
 559   int check_cap(const std::string& cap, uint32_t perm) const;
 560   bool is_valid_cap_type(const std::string& tp);
 561   void dump(Formatter *f) const;
 562   void dump(Formatter *f, const char *name) const;
 563
 564   void decode_json(JSONObj *obj);
 565 };
 566 WRITE_CLASS_ENCODER(RGWUserCaps)
 567
 568 void encode_json(const char *name, const obj_version& v, Formatter *f);
 569 void encode_json(const char *name, const RGWUserCaps& val, Formatter *f);
 570
 571 void decode_json_obj(obj_version& v, JSONObj *obj);
 572
 573
 574
 575 enum RGWIdentityType
 576 {
 577   TYPE_NONE=0,
 578   TYPE_RGW=1,
 579   TYPE_KEYSTONE=2,
 580   TYPE_LDAP=3,
 581   TYPE_ROLE=4,
 582   TYPE_WEB=5,
 583 };
 584
 585 static std::string RGW_STORAGE_CLASS_STANDARD = "STANDARD";
 586
 587 struct rgw_placement_rule {
 588   std::string name;
 589   std::string storage_class;
 590
 591   rgw_placement_rule() {}
 592   rgw_placement_rule(const std::string& _n, const std::string& _sc) : name(_n), storage_class(_sc) {}
 593   rgw_placement_rule(const rgw_placement_rule& _r, const std::string& _sc) : name(_r.name) {
 594     if (!_sc.empty()) {
 595       storage_class = _sc;
 596     } else {
 597       storage_class = _r.storage_class;
 598     }
 599   }
 600
 601   bool empty() const {
 602     return name.empty() && storage_class.empty();
 603   }
 604
 605   void inherit_from(const rgw_placement_rule& r) {
 606     if (name.empty()) {
 607       name = r.name;
 608     }
 609     if (storage_class.empty()) {
 610       storage_class = r.storage_class;
 611     }
 612   }
 613
 614   void clear() {
 615     name.clear();
 616     storage_class.clear();
 617   }
 618
 619   void init(const std::string& n, const std::string& c) {
 620     name = n;
 621     storage_class = c;
 622   }
 623
 624   static const std::string& get_canonical_storage_class(const std::string& storage_class) {
 625     if (storage_class.empty()) {
 626       return RGW_STORAGE_CLASS_STANDARD;
 627     }
 628     return storage_class;
 629   }
 630
 631   const std::string& get_storage_class() const {
 632     return get_canonical_storage_class(storage_class);
 633   }
 634
 635   int compare(const rgw_placement_rule& r) const {
 636     int c = name.compare(r.name);
 637     if (c != 0) {
 638       return c;
 639     }
 640     return get_storage_class().compare(r.get_storage_class());
 641   }
 642
 643   bool operator==(const rgw_placement_rule& r) const {
 644     return (name == r.name &&
 645             get_storage_class() == r.get_storage_class());
 646   }
 647
 648   bool operator!=(const rgw_placement_rule& r) const {
 649     return !(*this == r);
 650   }
 651
 652   void encode(bufferlist& bl) const {
 653     /* no ENCODE_START/END due to backward compatibility */
 654     std::string s = to_str();
 655     ceph::encode(s, bl);
 656   }
 657
 658   void decode(bufferlist::const_iterator& bl) {
 659     std::string s;
 660     ceph::decode(s, bl);
 661     from_str(s);
 662   }
 663
 664   std::string to_str() const {
 665     if (standard_storage_class()) {
 666       return name;
 667     }
 668     return to_str_explicit();
 669   }
 670
 671   std::string to_str_explicit() const {
 672     return name + "/" + storage_class;
 673   }
 674
 675   void from_str(const std::string& s) {
 676     size_t pos = s.find("/");
 677     if (pos == std::string::npos) {
 678       name = s;
 679       storage_class.clear();
 680       return;
 681     }
 682     name = s.substr(0, pos);
 683     storage_class = s.substr(pos + 1);
 684   }
 685
 686   bool standard_storage_class() const {
 687     return storage_class.empty() || storage_class == RGW_STORAGE_CLASS_STANDARD;
 688   }
 689 };
 690 WRITE_CLASS_ENCODER(rgw_placement_rule)
 691
 692 void encode_json(const char *name, const rgw_placement_rule& val, ceph::Formatter *f);
 693 void decode_json_obj(rgw_placement_rule& v, JSONObj *obj);
 694
 695 inline std::ostream& operator<<(std::ostream& out, const rgw_placement_rule& rule) {
 696   return out << rule.to_str();
 697 }
 698
 699 class RateLimiter;
 700 struct RGWRateLimitInfo {
 701   int64_t max_write_ops;
 702   int64_t max_read_ops;
 703   int64_t max_write_bytes;
 704   int64_t max_read_bytes;
 705   bool enabled = false;
 706   RGWRateLimitInfo()
 707     : max_write_ops(0), max_read_ops(0), max_write_bytes(0), max_read_bytes(0)  {}
 708
 709   void encode(bufferlist& bl) const {
 710     ENCODE_START(1, 1, bl);
 711     encode(max_write_ops, bl);
 712     encode(max_read_ops, bl);
 713     encode(max_write_bytes, bl);
 714     encode(max_read_bytes, bl);
 715     encode(enabled, bl);
 716     ENCODE_FINISH(bl);
 717   }
 718   void decode(bufferlist::const_iterator& bl) {
 719     DECODE_START(1, bl);
 720     decode(max_write_ops,bl);
 721     decode(max_read_ops, bl);
 722     decode(max_write_bytes,bl);
 723     decode(max_read_bytes, bl);
 724     decode(enabled, bl);
 725     DECODE_FINISH(bl);
 726   }
 727
 728   void dump(Formatter *f) const;
 729
 730   void decode_json(JSONObj *obj);
 731
 732 };
 733 WRITE_CLASS_ENCODER(RGWRateLimitInfo)
 734
 735 struct RGWUserInfo
 736 {
 737   rgw_user user_id;
 738   std::string display_name;
 739   std::string user_email;
 740   std::map<std::string, RGWAccessKey> access_keys;
 741   std::map<std::string, RGWAccessKey> swift_keys;
 742   std::map<std::string, RGWSubUser> subusers;
 743   __u8 suspended;
 744   int32_t max_buckets;
 745   uint32_t op_mask;
 746   RGWUserCaps caps;
 747   __u8 admin;
 748   __u8 system;
 749   rgw_placement_rule default_placement;
 750   std::list<std::string> placement_tags;
 751   RGWQuotaInfo bucket_quota;
 752   std::map<int, std::string> temp_url_keys;
 753   RGWQuotaInfo user_quota;
 754   uint32_t type;
 755   std::set<std::string> mfa_ids;
 756   std::string assumed_role_arn;
 757
 758   RGWUserInfo()
 759     : suspended(0),
 760       max_buckets(RGW_DEFAULT_MAX_BUCKETS),
 761       op_mask(RGW_OP_TYPE_ALL),
 762       admin(0),
 763       system(0),
 764       type(TYPE_NONE) {
 765   }
 766
 767   RGWAccessKey* get_key(const std::string& access_key) {
 768     if (access_keys.empty())
 769       return nullptr;
 770
 771     auto k = access_keys.find(access_key);
 772     if (k == access_keys.end())
 773       return nullptr;
 774     else
 775       return &(k->second);
 776   }
 777
 778   void encode(bufferlist& bl) const {
 779      ENCODE_START(22, 9, bl);
 780      encode((uint64_t)0, bl); // old auid
 781      std::string access_key;
 782      std::string secret_key;
 783      if (!access_keys.empty()) {
 784        std::map<std::string, RGWAccessKey>::const_iterator iter = access_keys.begin();
 785        const RGWAccessKey& k = iter->second;
 786        access_key = k.id;
 787        secret_key = k.key;
 788      }
 789      encode(access_key, bl);
 790      encode(secret_key, bl);
 791      encode(display_name, bl);
 792      encode(user_email, bl);
 793      std::string swift_name;
 794      std::string swift_key;
 795      if (!swift_keys.empty()) {
 796        std::map<std::string, RGWAccessKey>::const_iterator iter = swift_keys.begin();
 797        const RGWAccessKey& k = iter->second;
 798        swift_name = k.id;
 799        swift_key = k.key;
 800      }
 801      encode(swift_name, bl);
 802      encode(swift_key, bl);
 803      encode(user_id.id, bl);
 804      encode(access_keys, bl);
 805      encode(subusers, bl);
 806      encode(suspended, bl);
 807      encode(swift_keys, bl);
 808      encode(max_buckets, bl);
 809      encode(caps, bl);
 810      encode(op_mask, bl);
 811      encode(system, bl);
 812      encode(default_placement, bl);
 813      encode(placement_tags, bl);
 814      encode(bucket_quota, bl);
 815      encode(temp_url_keys, bl);
 816      encode(user_quota, bl);
 817      encode(user_id.tenant, bl);
 818      encode(admin, bl);
 819      encode(type, bl);
 820      encode(mfa_ids, bl);
 821      encode(assumed_role_arn, bl);
 822      encode(user_id.ns, bl);
 823      ENCODE_FINISH(bl);
 824   }
 825   void decode(bufferlist::const_iterator& bl) {
 826      DECODE_START_LEGACY_COMPAT_LEN_32(22, 9, 9, bl);
 827      if (struct_v >= 2) {
 828        uint64_t old_auid;
 829        decode(old_auid, bl);
 830      }
 831      std::string access_key;
 832      std::string secret_key;
 833     decode(access_key, bl);
 834     decode(secret_key, bl);
 835     if (struct_v < 6) {
 836       RGWAccessKey k;
 837       k.id = access_key;
 838       k.key = secret_key;
 839       access_keys[access_key] = k;
 840     }
 841     decode(display_name, bl);
 842     decode(user_email, bl);
 843     /* We populate swift_keys map later nowadays, but we have to decode. */
 844     std::string swift_name;
 845     std::string swift_key;
 846     if (struct_v >= 3) decode(swift_name, bl);
 847     if (struct_v >= 4) decode(swift_key, bl);
 848     if (struct_v >= 5)
 849       decode(user_id.id, bl);
 850     else
 851       user_id.id = access_key;
 852     if (struct_v >= 6) {
 853       decode(access_keys, bl);
 854       decode(subusers, bl);
 855     }
 856     suspended = 0;
 857     if (struct_v >= 7) {
 858       decode(suspended, bl);
 859     }
 860     if (struct_v >= 8) {
 861       decode(swift_keys, bl);
 862     }
 863     if (struct_v >= 10) {
 864       decode(max_buckets, bl);
 865     } else {
 866       max_buckets = RGW_DEFAULT_MAX_BUCKETS;
 867     }
 868     if (struct_v >= 11) {
 869       decode(caps, bl);
 870     }
 871     if (struct_v >= 12) {
 872       decode(op_mask, bl);
 873     } else {
 874       op_mask = RGW_OP_TYPE_ALL;
 875     }
 876     if (struct_v >= 13) {
 877       decode(system, bl);
 878       decode(default_placement, bl);
 879       decode(placement_tags, bl); /* tags of allowed placement rules */
 880     }
 881     if (struct_v >= 14) {
 882       decode(bucket_quota, bl);
 883     }
 884     if (struct_v >= 15) {
 885      decode(temp_url_keys, bl);
 886     }
 887     if (struct_v >= 16) {
 888       decode(user_quota, bl);
 889     }
 890     if (struct_v >= 17) {
 891       decode(user_id.tenant, bl);
 892     } else {
 893       user_id.tenant.clear();
 894     }
 895     if (struct_v >= 18) {
 896       decode(admin, bl);
 897     }
 898     if (struct_v >= 19) {
 899       decode(type, bl);
 900     }
 901     if (struct_v >= 20) {
 902       decode(mfa_ids, bl);
 903     }
 904     if (struct_v >= 21) {
 905       decode(assumed_role_arn, bl);
 906     }
 907     if (struct_v >= 22) {
 908       decode(user_id.ns, bl);
 909     } else {
 910       user_id.ns.clear();
 911     }
 912     DECODE_FINISH(bl);
 913   }
 914   void dump(Formatter *f) const;
 915   static void generate_test_instances(std::list<RGWUserInfo*>& o);
 916
 917   void decode_json(JSONObj *obj);
 918 };
 919 WRITE_CLASS_ENCODER(RGWUserInfo)
 920
 921 struct rgw_raw_obj {
 922   rgw_pool pool;
 923   std::string oid;
 924   std::string loc;
 925
 926   rgw_raw_obj() {}
 927   rgw_raw_obj(const rgw_pool& _pool, const std::string& _oid) {
 928     init(_pool, _oid);
 929   }
 930   rgw_raw_obj(const rgw_pool& _pool, const std::string& _oid, const std::string& _loc) : loc(_loc) {
 931     init(_pool, _oid);
 932   }
 933
 934   void init(const rgw_pool& _pool, const std::string& _oid) {
 935     pool = _pool;
 936     oid = _oid;
 937   }
 938
 939   bool empty() const {
 940     return oid.empty();
 941   }
 942
 943   void encode(bufferlist& bl) const {
 944      ENCODE_START(6, 6, bl);
 945     encode(pool, bl);
 946     encode(oid, bl);
 947     encode(loc, bl);
 948     ENCODE_FINISH(bl);
 949   }
 950
 951   void decode_from_rgw_obj(bufferlist::const_iterator& bl);
 952
 953   void decode(bufferlist::const_iterator& bl) {
 954     unsigned ofs = bl.get_off();
 955     DECODE_START(6, bl);
 956     if (struct_v < 6) {
 957       /*
 958        * this object was encoded as rgw_obj, prior to rgw_raw_obj been split out of it,
 959        * let's decode it as rgw_obj and convert it
 960        */
 961       bl.seek(ofs);
 962       decode_from_rgw_obj(bl);
 963       return;
 964     }
 965     decode(pool, bl);
 966     decode(oid, bl);
 967     decode(loc, bl);
 968     DECODE_FINISH(bl);
 969   }
 970
 971   bool operator<(const rgw_raw_obj& o) const {
 972     int r = pool.compare(o.pool);
 973     if (r == 0) {
 974       r = oid.compare(o.oid);
 975       if (r == 0) {
 976         r = loc.compare(o.loc);
 977       }
 978     }
 979     return (r < 0);
 980   }
 981
 982   bool operator==(const rgw_raw_obj& o) const {
 983     return (pool == o.pool && oid == o.oid && loc == o.loc);
 984   }
 985
 986   void dump(Formatter *f) const;
 987   void decode_json(JSONObj *obj);
 988 };
 989 WRITE_CLASS_ENCODER(rgw_raw_obj)
 990
 991 inline std::ostream& operator<<(std::ostream& out, const rgw_raw_obj& o) {
 992   out << o.pool << ":" << o.oid;
 993   return out;
 994 }
 995
 996 struct rgw_bucket_placement {
 997   rgw_placement_rule placement_rule;
 998   rgw_bucket bucket;
 999
1000   void dump(Formatter *f) const;
1001 };
1002
1003 struct RGWObjVersionTracker {
1004   obj_version read_version;
1005   obj_version write_version;
1006
1007   obj_version *version_for_read() {
1008     return &read_version;
1009   }
1010
1011   obj_version *version_for_write() {
1012     if (write_version.ver == 0)
1013       return NULL;
1014
1015     return &write_version;
1016   }
1017
1018   obj_version *version_for_check() {
1019     if (read_version.ver == 0)
1020       return NULL;
1021
1022     return &read_version;
1023   }
1024
1025   void prepare_op_for_read(librados::ObjectReadOperation *op);
1026   void prepare_op_for_write(librados::ObjectWriteOperation *op);
1027
1028   void apply_write();
1029
1030   void clear() {
1031     read_version = obj_version();
1032     write_version = obj_version();
1033   }
1034
1035   void generate_new_write_ver(CephContext *cct);
1036 };
1037
1038 inline std::ostream& operator<<(std::ostream& out, const obj_version &v)
1039 {
1040   out << v.tag << ":" << v.ver;
1041   return out;
1042 }
1043
1044 inline std::ostream& operator<<(std::ostream& out, const RGWObjVersionTracker &ot)
1045 {
1046   out << "{r=" << ot.read_version << ",w=" << ot.write_version << "}";
1047   return out;
1048 }
1049
1050 enum RGWBucketFlags {
1051   BUCKET_SUSPENDED = 0x1,
1052   BUCKET_VERSIONED = 0x2,
1053   BUCKET_VERSIONS_SUSPENDED = 0x4,
1054   BUCKET_DATASYNC_DISABLED = 0X8,
1055   BUCKET_MFA_ENABLED = 0X10,
1056   BUCKET_OBJ_LOCK_ENABLED = 0X20,
1057 };
1058
1059 class RGWSI_Zone;
1060
1061 struct RGWBucketInfo {
1062   rgw_bucket bucket;
1063   rgw_user owner;
1064   uint32_t flags{0};
1065   std::string zonegroup;
1066   ceph::real_time creation_time;
1067   rgw_placement_rule placement_rule;
1068   bool has_instance_obj{false};
1069   RGWObjVersionTracker objv_tracker; /* we don't need to serialize this, for runtime tracking */
1070   RGWQuotaInfo quota;
1071
1072   // layout of bucket index objects
1073   rgw::BucketLayout layout;
1074
1075   // Represents the number of bucket index object shards:
1076   //   - value of 0 indicates there is no sharding (this is by default
1077   //     before this feature is implemented).
1078   //   - value of UINT32_T::MAX indicates this is a blind bucket.
1079
1080   // Represents the shard number for blind bucket.
1081   const static uint32_t NUM_SHARDS_BLIND_BUCKET;
1082
1083   bool requester_pays{false};
1084
1085   bool has_website{false};
1086   RGWBucketWebsiteConf website_conf;
1087
1088   bool swift_versioning{false};
1089   std::string swift_ver_location;
1090
1091   std::map<std::string, uint32_t> mdsearch_config;
1092
1093   // resharding
1094   cls_rgw_reshard_status reshard_status{cls_rgw_reshard_status::NOT_RESHARDING};
1095   std::string new_bucket_instance_id;
1096
1097   RGWObjectLock obj_lock;
1098
1099   std::optional<rgw_sync_policy_info> sync_policy;
1100
1101   void encode(bufferlist& bl) const;
1102   void decode(bufferlist::const_iterator& bl);
1103
1104   void dump(Formatter *f) const;
1105   static void generate_test_instances(std::list<RGWBucketInfo*>& o);
1106
1107   void decode_json(JSONObj *obj);
1108
1109   bool versioned() const { return (flags & BUCKET_VERSIONED) != 0; }
1110   int versioning_status() const { return flags & (BUCKET_VERSIONED | BUCKET_VERSIONS_SUSPENDED | BUCKET_MFA_ENABLED); }
1111   bool versioning_enabled() const { return (versioning_status() & (BUCKET_VERSIONED | BUCKET_VERSIONS_SUSPENDED)) == BUCKET_VERSIONED; }
1112   bool mfa_enabled() const { return (versioning_status() & BUCKET_MFA_ENABLED) != 0; }
1113   bool datasync_flag_enabled() const { return (flags & BUCKET_DATASYNC_DISABLED) == 0; }
1114   bool obj_lock_enabled() const { return (flags & BUCKET_OBJ_LOCK_ENABLED) != 0; }
1115
1116   bool has_swift_versioning() const {
1117     /* A bucket may be versioned through one mechanism only. */
1118     return swift_versioning && !versioned();
1119   }
1120
1121   void set_sync_policy(rgw_sync_policy_info&& policy);
1122
1123   bool empty_sync_policy() const;
1124
1125   RGWBucketInfo();
1126   ~RGWBucketInfo();
1127 };
1128 WRITE_CLASS_ENCODER(RGWBucketInfo)
1129
1130 struct RGWBucketEntryPoint
1131 {
1132   rgw_bucket bucket;
1133   rgw_user owner;
1134   ceph::real_time creation_time;
1135   bool linked;
1136
1137   bool has_bucket_info;
1138   RGWBucketInfo old_bucket_info;
1139
1140   RGWBucketEntryPoint() : linked(false), has_bucket_info(false) {}
1141
1142   void encode(bufferlist& bl) const {
1143     ENCODE_START(10, 8, bl);
1144     encode(bucket, bl);
1145     encode(owner.id, bl);
1146     encode(linked, bl);
1147     uint64_t ctime = (uint64_t)real_clock::to_time_t(creation_time);
1148     encode(ctime, bl);
1149     encode(owner, bl);
1150     encode(creation_time, bl);
1151     ENCODE_FINISH(bl);
1152   }
1153   void decode(bufferlist::const_iterator& bl) {
1154     auto orig_iter = bl;
1155     DECODE_START_LEGACY_COMPAT_LEN_32(10, 4, 4, bl);
1156     if (struct_v < 8) {
1157       /* ouch, old entry, contains the bucket info itself */
1158       old_bucket_info.decode(orig_iter);
1159       has_bucket_info = true;
1160       return;
1161     }
1162     has_bucket_info = false;
1163     decode(bucket, bl);
1164     decode(owner.id, bl);
1165     decode(linked, bl);
1166     uint64_t ctime;
1167     decode(ctime, bl);
1168     if (struct_v < 10) {
1169       creation_time = real_clock::from_time_t((time_t)ctime);
1170     }
1171     if (struct_v >= 9) {
1172       decode(owner, bl);
1173     }
1174     if (struct_v >= 10) {
1175       decode(creation_time, bl);
1176     }
1177     DECODE_FINISH(bl);
1178   }
1179
1180   void dump(Formatter *f) const;
1181   void decode_json(JSONObj *obj);
1182   static void generate_test_instances(std::list<RGWBucketEntryPoint*>& o);
1183 };
1184 WRITE_CLASS_ENCODER(RGWBucketEntryPoint)
1185
1186 struct RGWStorageStats
1187 {
1188   RGWObjCategory category;
1189   uint64_t size;
1190   uint64_t size_rounded;
1191   uint64_t num_objects;
1192   uint64_t size_utilized{0}; //< size after compression, encryption
1193   bool dump_utilized;        // whether dump should include utilized values
1194
1195   RGWStorageStats(bool _dump_utilized=true)
1196     : category(RGWObjCategory::None),
1197       size(0),
1198       size_rounded(0),
1199       num_objects(0),
1200       dump_utilized(_dump_utilized)
1201   {}
1202
1203   void dump(Formatter *f) const;
1204 }; // RGWStorageStats
1205
1206 class RGWEnv;
1207
1208 /* Namespaced forward declarations. */
1209 namespace rgw {
1210   namespace auth {
1211     namespace s3 {
1212       class AWSBrowserUploadAbstractor;
1213       class STSEngine;
1214     }
1215     class Completer;
1216   }
1217   namespace io {
1218     class BasicClient;
1219   }
1220 }
1221
1222 using meta_map_t = boost::container::flat_map <std::string, std::string>;
1223
1224 struct req_info {
1225   const RGWEnv *env;
1226   RGWHTTPArgs args;
1227   meta_map_t x_meta_map;
1228   meta_map_t crypt_attribute_map;
1229
1230   std::string host;
1231   const char *method;
1232   std::string script_uri;
1233   std::string request_uri;
1234   std::string request_uri_aws4;
1235   std::string effective_uri;
1236   std::string request_params;
1237   std::string domain;
1238   std::string storage_class;
1239
1240   req_info(CephContext *cct, const RGWEnv *env);
1241   void rebuild_from(req_info& src);
1242   void init_meta_info(const DoutPrefixProvider *dpp, bool *found_bad_meta);
1243 };
1244
1245 typedef cls_rgw_obj_key rgw_obj_index_key;
1246
1247 struct rgw_obj_key {
1248   std::string name;
1249   std::string instance;
1250   std::string ns;
1251
1252   rgw_obj_key() {}
1253   // cppcheck-suppress noExplicitConstructor
1254   rgw_obj_key(const std::string& n) : name(n) {}
1255   rgw_obj_key(const std::string& n, const std::string& i) : name(n), instance(i) {}
1256   rgw_obj_key(const std::string& n, const std::string& i, const std::string& _ns) : name(n), instance(i), ns(_ns) {}
1257
1258   rgw_obj_key(const rgw_obj_index_key& k) {
1259     parse_index_key(k.name, &name, &ns);
1260     instance = k.instance;
1261   }
1262
1263   static void parse_index_key(const std::string& key, std::string *name, std::string *ns) {
1264     if (key[0] != '_') {
1265       *name = key;
1266       ns->clear();
1267       return;
1268     }
1269     if (key[1] == '_') {
1270       *name = key.substr(1);
1271       ns->clear();
1272       return;
1273     }
1274     ssize_t pos = key.find('_', 1);
1275     if (pos < 0) {
1276       /* shouldn't happen, just use key */
1277       *name = key;
1278       ns->clear();
1279       return;
1280     }
1281
1282     *name = key.substr(pos + 1);
1283     *ns = key.substr(1, pos -1);
1284   }
1285
1286   void set(const std::string& n) {
1287     name = n;
1288     instance.clear();
1289     ns.clear();
1290   }
1291
1292   void set(const std::string& n, const std::string& i) {
1293     name = n;
1294     instance = i;
1295     ns.clear();
1296   }
1297
1298   void set(const std::string& n, const std::string& i, const std::string& _ns) {
1299     name = n;
1300     instance = i;
1301     ns = _ns;
1302   }
1303
1304   bool set(const rgw_obj_index_key& index_key) {
1305     if (!parse_raw_oid(index_key.name, this)) {
1306       return false;
1307     }
1308     instance = index_key.instance;
1309     return true;
1310   }
1311
1312   void set_instance(const std::string& i) {
1313     instance = i;
1314   }
1315
1316   const std::string& get_instance() const {
1317     return instance;
1318   }
1319
1320   void set_ns(const std::string& _ns) {
1321     ns = _ns;
1322   }
1323
1324   const std::string& get_ns() const {
1325     return ns;
1326   }
1327
1328   std::string get_index_key_name() const {
1329     if (ns.empty()) {
1330       if (name.size() < 1 || name[0] != '_') {
1331         return name;
1332       }
1333       return std::string("_") + name;
1334     };
1335
1336     char buf[ns.size() + 16];
1337     snprintf(buf, sizeof(buf), "_%s_", ns.c_str());
1338     return std::string(buf) + name;
1339   };
1340
1341   void get_index_key(rgw_obj_index_key *key) const {
1342     key->name = get_index_key_name();
1343     key->instance = instance;
1344   }
1345
1346   std::string get_loc() const {
1347     /*
1348      * For backward compatibility. Older versions used to have object locator on all objects,
1349      * however, the name was the effective object locator. This had the same effect as not
1350      * having object locator at all for most objects but the ones that started with underscore as
1351      * these were escaped.
1352      */
1353     if (name[0] == '_' && ns.empty()) {
1354       return name;
1355     }
1356
1357     return {};
1358   }
1359
1360   bool empty() const {
1361     return name.empty();
1362   }
1363
1364   bool have_null_instance() const {
1365     return instance == "null";
1366   }
1367
1368   bool have_instance() const {
1369     return !instance.empty();
1370   }
1371
1372   bool need_to_encode_instance() const {
1373     return have_instance() && !have_null_instance();
1374   }
1375
1376   std::string get_oid() const {
1377     if (ns.empty() && !need_to_encode_instance()) {
1378       if (name.size() < 1 || name[0] != '_') {
1379         return name;
1380       }
1381       return std::string("_") + name;
1382     }
1383
1384     std::string oid = "_";
1385     oid.append(ns);
1386     if (need_to_encode_instance()) {
1387       oid.append(std::string(":") + instance);
1388     }
1389     oid.append("_");
1390     oid.append(name);
1391     return oid;
1392   }
1393
1394   bool operator==(const rgw_obj_key& k) const {
1395     return (name.compare(k.name) == 0) &&
1396            (instance.compare(k.instance) == 0);
1397   }
1398
1399   bool operator<(const rgw_obj_key& k) const {
1400     int r = name.compare(k.name);
1401     if (r == 0) {
1402       r = instance.compare(k.instance);
1403     }
1404     return (r < 0);
1405   }
1406
1407   bool operator<=(const rgw_obj_key& k) const {
1408     return !(k < *this);
1409   }
1410
1411   static void parse_ns_field(std::string& ns, std::string& instance) {
1412     int pos = ns.find(':');
1413     if (pos >= 0) {
1414       instance = ns.substr(pos + 1);
1415       ns = ns.substr(0, pos);
1416     } else {
1417       instance.clear();
1418     }
1419   }
1420
1421   // takes an oid and parses out the namespace (ns), name, and
1422   // instance
1423   static bool parse_raw_oid(const std::string& oid, rgw_obj_key *key) {
1424     key->instance.clear();
1425     key->ns.clear();
1426     if (oid[0] != '_') {
1427       key->name = oid;
1428       return true;
1429     }
1430
1431     if (oid.size() >= 2 && oid[1] == '_') {
1432       key->name = oid.substr(1);
1433       return true;
1434     }
1435
1436     if (oid.size() < 3) // for namespace, min size would be 3: _x_
1437       return false;
1438
1439     size_t pos = oid.find('_', 2); // oid must match ^_[^_].+$
1440     if (pos == std::string::npos)
1441       return false;
1442
1443     key->ns = oid.substr(1, pos - 1);
1444     parse_ns_field(key->ns, key->instance);
1445
1446     key->name = oid.substr(pos + 1);
1447     return true;
1448   }
1449
1450   /**
1451    * Translate a namespace-mangled object name to the user-facing name
1452    * existing in the given namespace.
1453    *
1454    * If the object is part of the given namespace, it returns true
1455    * and cuts down the name to the unmangled version. If it is not
1456    * part of the given namespace, it returns false.
1457    */
1458   static bool oid_to_key_in_ns(const std::string& oid, rgw_obj_key *key, const std::string& ns) {
1459     bool ret = parse_raw_oid(oid, key);
1460     if (!ret) {
1461       return ret;
1462     }
1463
1464     return (ns == key->ns);
1465   }
1466
1467   /**
1468    * Given a mangled object name and an empty namespace std::string, this
1469    * function extracts the namespace into the std::string and sets the object
1470    * name to be the unmangled version.
1471    *
1472    * It returns true after successfully doing so, or
1473    * false if it fails.
1474    */
1475   static bool strip_namespace_from_name(std::string& name, std::string& ns, std::string& instance) {
1476     ns.clear();
1477     instance.clear();
1478     if (name[0] != '_') {
1479       return true;
1480     }
1481
1482     size_t pos = name.find('_', 1);
1483     if (pos == std::string::npos) {
1484       return false;
1485     }
1486
1487     if (name[1] == '_') {
1488       name = name.substr(1);
1489       return true;
1490     }
1491
1492     size_t period_pos = name.find('.');
1493     if (period_pos < pos) {
1494       return false;
1495     }
1496
1497     ns = name.substr(1, pos-1);
1498     name = name.substr(pos+1, std::string::npos);
1499
1500     parse_ns_field(ns, instance);
1501     return true;
1502   }
1503
1504   void encode(bufferlist& bl) const {
1505     ENCODE_START(2, 1, bl);
1506     encode(name, bl);
1507     encode(instance, bl);
1508     encode(ns, bl);
1509     ENCODE_FINISH(bl);
1510   }
1511   void decode(bufferlist::const_iterator& bl) {
1512     DECODE_START(2, bl);
1513     decode(name, bl);
1514     decode(instance, bl);
1515     if (struct_v >= 2) {
1516       decode(ns, bl);
1517     }
1518     DECODE_FINISH(bl);
1519   }
1520   void dump(Formatter *f) const;
1521   void decode_json(JSONObj *obj);
1522
1523   std::string to_str() const {
1524     if (instance.empty()) {
1525       return name;
1526     }
1527     char buf[name.size() + instance.size() + 16];
1528     snprintf(buf, sizeof(buf), "%s[%s]", name.c_str(), instance.c_str());
1529     return buf;
1530   }
1531 };
1532 WRITE_CLASS_ENCODER(rgw_obj_key)
1533
1534 inline std::ostream& operator<<(std::ostream& out, const rgw_obj_key &o) {
1535   return out << o.to_str();
1536 }
1537
1538 struct req_init_state {
1539   /* Keeps [[tenant]:]bucket until we parse the token. */
1540   std::string url_bucket;
1541   std::string src_bucket;
1542 };
1543
1544 #include "rgw_auth.h"
1545
1546 class RGWObjectCtx;
1547 class RGWSysObjectCtx;
1548
1549 /** Store all the state necessary to complete and respond to an HTTP request*/
1550 struct req_state : DoutPrefixProvider {
1551   CephContext *cct;
1552   rgw::io::BasicClient *cio{nullptr};
1553   http_op op{OP_UNKNOWN};
1554   RGWOpType op_type{};
1555   std::shared_ptr<RateLimiter> ratelimit_data;
1556   RGWRateLimitInfo user_ratelimit;
1557   RGWRateLimitInfo bucket_ratelimit;
1558   std::string ratelimit_bucket_marker;
1559   std::string ratelimit_user_name;
1560   bool content_started{false};
1561   int format{0};
1562   ceph::Formatter *formatter{nullptr};
1563   std::string decoded_uri;
1564   std::string relative_uri;
1565   const char *length{nullptr};
1566   int64_t content_length{0};
1567   std::map<std::string, std::string> generic_attrs;
1568   rgw_err err;
1569   bool expect_cont{false};
1570   uint64_t obj_size{0};
1571   bool enable_ops_log;
1572   bool enable_usage_log;
1573   uint8_t defer_to_bucket_acls;
1574   uint32_t perm_mask{0};
1575
1576   /* Set once when url_bucket is parsed and not violated thereafter. */
1577   std::string account_name;
1578
1579   std::string bucket_tenant;
1580   std::string bucket_name;
1581
1582   /* bucket is only created in rgw_build_bucket_policies() and should never be
1583    * overwritten */
1584   std::unique_ptr<rgw::sal::Bucket> bucket;
1585   std::unique_ptr<rgw::sal::Object> object;
1586   std::string src_tenant_name;
1587   std::string src_bucket_name;
1588   std::unique_ptr<rgw::sal::Object> src_object;
1589   ACLOwner bucket_owner;
1590   ACLOwner owner;
1591
1592   std::string zonegroup_name;
1593   std::string zonegroup_endpoint;
1594   std::string bucket_instance_id;
1595   int bucket_instance_shard_id{-1};
1596   std::string redirect_zone_endpoint;
1597
1598   std::string redirect;
1599
1600   real_time bucket_mtime;
1601   std::map<std::string, ceph::bufferlist> bucket_attrs;
1602   bool bucket_exists{false};
1603   rgw_placement_rule dest_placement;
1604
1605   bool has_bad_meta{false};
1606
1607   std::unique_ptr<rgw::sal::User> user;
1608
1609   struct {
1610     /* TODO(rzarzynski): switch out to the static_ptr for both members. */
1611
1612     /* Object having the knowledge about an authenticated identity and allowing
1613      * to apply it during the authorization phase (verify_permission() methods
1614      * of a given RGWOp). Thus, it bounds authentication and authorization steps
1615      * through a well-defined interface. For more details, see rgw_auth.h. */
1616     std::unique_ptr<rgw::auth::Identity> identity;
1617
1618     std::shared_ptr<rgw::auth::Completer> completer;
1619
1620     /* A container for credentials of the S3's browser upload. It's necessary
1621      * because: 1) the ::authenticate() method of auth engines and strategies
1622      * take req_state only; 2) auth strategies live much longer than RGWOps -
1623      * there is no way to pass additional data dependencies through ctors. */
1624     class {
1625       /* Writer. */
1626       friend class RGWPostObj_ObjStore_S3;
1627       /* Reader. */
1628       friend class rgw::auth::s3::AWSBrowserUploadAbstractor;
1629       friend class rgw::auth::s3::STSEngine;
1630
1631       std::string access_key;
1632       std::string signature;
1633       std::string x_amz_algorithm;
1634       std::string x_amz_credential;
1635       std::string x_amz_date;
1636       std::string x_amz_security_token;
1637       ceph::bufferlist encoded_policy;
1638     } s3_postobj_creds;
1639   } auth;
1640
1641   std::unique_ptr<RGWAccessControlPolicy> user_acl;
1642   std::unique_ptr<RGWAccessControlPolicy> bucket_acl;
1643   std::unique_ptr<RGWAccessControlPolicy> object_acl;
1644
1645   rgw::IAM::Environment env;
1646   boost::optional<rgw::IAM::Policy> iam_policy;
1647   boost::optional<PublicAccessBlockConfiguration> bucket_access_conf;
1648   std::vector<rgw::IAM::Policy> iam_user_policies;
1649
1650   /* Is the request made by an user marked as a system one?
1651    * Being system user means we also have the admin status. */
1652   bool system_request{false};
1653
1654   std::string canned_acl;
1655   bool has_acl_header{false};
1656   bool local_source{false}; /* source is local */
1657
1658   int prot_flags{0};
1659
1660   /* Content-Disposition override for TempURL of Swift API. */
1661   struct {
1662     std::string override;
1663     std::string fallback;
1664   } content_disp;
1665
1666   std::string host_id;
1667
1668   req_info info;
1669   req_init_state init_state;
1670
1671   using Clock = ceph::coarse_real_clock;
1672   Clock::time_point time;
1673
1674   Clock::duration time_elapsed() const { return Clock::now() - time; }
1675
1676   RGWObjectCtx *obj_ctx{nullptr};
1677   std::string dialect;
1678   std::string req_id;
1679   std::string trans_id;
1680   uint64_t id;
1681
1682   RGWObjTags tagset;
1683
1684   bool mfa_verified{false};
1685
1686   /// optional coroutine context
1687   optional_yield yield{null_yield};
1688
1689   //token claims from STS token for ops log (can be used for Keystone token also)
1690   std::vector<std::string> token_claims;
1691
1692   std::vector<rgw::IAM::Policy> session_policies;
1693
1694   jspan trace;
1695
1696   //Principal tags that come in as part of AssumeRoleWithWebIdentity
1697   std::vector<std::pair<std::string, std::string>> principal_tags;
1698
1699   req_state(CephContext* _cct, RGWEnv* e, uint64_t id);
1700   ~req_state();
1701
1702
1703   void set_user(std::unique_ptr<rgw::sal::User>& u) { user.swap(u); }
1704   bool is_err() const { return err.is_err(); }
1705
1706   // implements DoutPrefixProvider
1707   std::ostream& gen_prefix(std::ostream& out) const override;
1708   CephContext* get_cct() const override { return cct; }
1709   unsigned get_subsys() const override { return ceph_subsys_rgw; }
1710 };
1711
1712 void set_req_state_err(struct req_state*, int);
1713 void set_req_state_err(struct req_state*, int, const std::string&);
1714 void set_req_state_err(struct rgw_err&, int, const int);
1715 void dump(struct req_state*);
1716
1717 /** Store basic data on bucket */
1718 struct RGWBucketEnt {
1719   rgw_bucket bucket;
1720   size_t size;
1721   size_t size_rounded;
1722   ceph::real_time creation_time;
1723   uint64_t count;
1724
1725   /* The placement_rule is necessary to calculate per-storage-policy statics
1726    * of the Swift API. Although the info available in RGWBucketInfo, we need
1727    * to duplicate it here to not affect the performance of buckets listing. */
1728   rgw_placement_rule placement_rule;
1729
1730   RGWBucketEnt()
1731     : size(0),
1732       size_rounded(0),
1733       count(0) {
1734   }
1735   RGWBucketEnt(const RGWBucketEnt&) = default;
1736   RGWBucketEnt(RGWBucketEnt&&) = default;
1737   explicit RGWBucketEnt(const rgw_user& u, cls_user_bucket_entry&& e)
1738     : bucket(u, std::move(e.bucket)),
1739       size(e.size),
1740       size_rounded(e.size_rounded),
1741       creation_time(e.creation_time),
1742       count(e.count) {
1743   }
1744
1745   RGWBucketEnt& operator=(const RGWBucketEnt&) = default;
1746
1747   void convert(cls_user_bucket_entry *b) const {
1748     bucket.convert(&b->bucket);
1749     b->size = size;
1750     b->size_rounded = size_rounded;
1751     b->creation_time = creation_time;
1752     b->count = count;
1753   }
1754
1755   void encode(bufferlist& bl) const {
1756     ENCODE_START(7, 5, bl);
1757     uint64_t s = size;
1758     __u32 mt = ceph::real_clock::to_time_t(creation_time);
1759     std::string empty_str;  // originally had the bucket name here, but we encode bucket later
1760     encode(empty_str, bl);
1761     encode(s, bl);
1762     encode(mt, bl);
1763     encode(count, bl);
1764     encode(bucket, bl);
1765     s = size_rounded;
1766     encode(s, bl);
1767     encode(creation_time, bl);
1768     encode(placement_rule, bl);
1769     ENCODE_FINISH(bl);
1770   }
1771   void decode(bufferlist::const_iterator& bl) {
1772     DECODE_START_LEGACY_COMPAT_LEN(7, 5, 5, bl);
1773     __u32 mt;
1774     uint64_t s;
1775     std::string empty_str;  // backward compatibility
1776     decode(empty_str, bl);
1777     decode(s, bl);
1778     decode(mt, bl);
1779     size = s;
1780     if (struct_v < 6) {
1781       creation_time = ceph::real_clock::from_time_t(mt);
1782     }
1783     if (struct_v >= 2)
1784       decode(count, bl);
1785     if (struct_v >= 3)
1786       decode(bucket, bl);
1787     if (struct_v >= 4)
1788       decode(s, bl);
1789     size_rounded = s;
1790     if (struct_v >= 6)
1791       decode(creation_time, bl);
1792     if (struct_v >= 7)
1793       decode(placement_rule, bl);
1794     DECODE_FINISH(bl);
1795   }
1796   void dump(Formatter *f) const;
1797   static void generate_test_instances(std::list<RGWBucketEnt*>& o);
1798 };
1799 WRITE_CLASS_ENCODER(RGWBucketEnt)
1800
1801 struct rgw_obj {
1802   rgw_bucket bucket;
1803   rgw_obj_key key;
1804
1805   bool in_extra_data{false}; /* in-memory only member, does not serialize */
1806
1807   // Represents the hash index source for this object once it is set (non-empty)
1808   std::string index_hash_source;
1809
1810   rgw_obj() {}
1811   rgw_obj(const rgw_bucket& b, const std::string& name) : bucket(b), key(name) {}
1812   rgw_obj(const rgw_bucket& b, const rgw_obj_key& k) : bucket(b), key(k) {}
1813   rgw_obj(const rgw_bucket& b, const rgw_obj_index_key& k) : bucket(b), key(k) {}
1814
1815   void init(const rgw_bucket& b, const std::string& name) {
1816     bucket = b;
1817     key.set(name);
1818   }
1819   void init(const rgw_bucket& b, const std::string& name, const std::string& i, const std::string& n) {
1820     bucket = b;
1821     key.set(name, i, n);
1822   }
1823   void init_ns(const rgw_bucket& b, const std::string& name, const std::string& n) {
1824     bucket = b;
1825     key.name = name;
1826     key.instance.clear();
1827     key.ns = n;
1828   }
1829
1830   bool empty() const {
1831     return key.empty();
1832   }
1833
1834   void set_key(const rgw_obj_key& k) {
1835     key = k;
1836   }
1837
1838   std::string get_oid() const {
1839     return key.get_oid();
1840   }
1841
1842   const std::string& get_hash_object() const {
1843     return index_hash_source.empty() ? key.name : index_hash_source;
1844   }
1845
1846   void set_in_extra_data(bool val) {
1847     in_extra_data = val;
1848   }
1849
1850   bool is_in_extra_data() const {
1851     return in_extra_data;
1852   }
1853
1854   void encode(bufferlist& bl) const {
1855     ENCODE_START(6, 6, bl);
1856     encode(bucket, bl);
1857     encode(key.ns, bl);
1858     encode(key.name, bl);
1859     encode(key.instance, bl);
1860 //    encode(placement_id, bl);
1861     ENCODE_FINISH(bl);
1862   }
1863   void decode(bufferlist::const_iterator& bl) {
1864     DECODE_START_LEGACY_COMPAT_LEN(6, 3, 3, bl);
1865     if (struct_v < 6) {
1866       std::string s;
1867       decode(bucket.name, bl); /* bucket.name */
1868       decode(s, bl); /* loc */
1869       decode(key.ns, bl);
1870       decode(key.name, bl);
1871       if (struct_v >= 2)
1872         decode(bucket, bl);
1873       if (struct_v >= 4)
1874         decode(key.instance, bl);
1875       if (key.ns.empty() && key.instance.empty()) {
1876         if (key.name[0] == '_') {
1877           key.name = key.name.substr(1);
1878         }
1879       } else {
1880         if (struct_v >= 5) {
1881           decode(key.name, bl);
1882         } else {
1883           ssize_t pos = key.name.find('_', 1);
1884           if (pos < 0) {
1885             throw buffer::malformed_input();
1886           }
1887           key.name = key.name.substr(pos + 1);
1888         }
1889       }
1890     } else {
1891       decode(bucket, bl);
1892       decode(key.ns, bl);
1893       decode(key.name, bl);
1894       decode(key.instance, bl);
1895 //      decode(placement_id, bl);
1896     }
1897     DECODE_FINISH(bl);
1898   }
1899   void dump(Formatter *f) const;
1900   static void generate_test_instances(std::list<rgw_obj*>& o);
1901
1902   bool operator==(const rgw_obj& o) const {
1903     return (key == o.key) &&
1904            (bucket == o.bucket);
1905   }
1906   bool operator<(const rgw_obj& o) const {
1907     int r = key.name.compare(o.key.name);
1908     if (r == 0) {
1909       r = bucket.bucket_id.compare(o.bucket.bucket_id); /* not comparing bucket.name, if bucket_id is equal so will be bucket.name */
1910       if (r == 0) {
1911         r = key.ns.compare(o.key.ns);
1912         if (r == 0) {
1913           r = key.instance.compare(o.key.instance);
1914         }
1915       }
1916     }
1917
1918     return (r < 0);
1919   }
1920
1921   const rgw_pool& get_explicit_data_pool() {
1922     if (!in_extra_data || bucket.explicit_placement.data_extra_pool.empty()) {
1923       return bucket.explicit_placement.data_pool;
1924     }
1925     return bucket.explicit_placement.data_extra_pool;
1926   }
1927 };
1928 WRITE_CLASS_ENCODER(rgw_obj)
1929
1930 struct rgw_cache_entry_info {
1931   std::string cache_locator;
1932   uint64_t gen;
1933
1934   rgw_cache_entry_info() : gen(0) {}
1935 };
1936
1937 inline std::ostream& operator<<(std::ostream& out, const rgw_obj &o) {
1938   return out << o.bucket.name << ":" << o.get_oid();
1939 }
1940
1941 struct multipart_upload_info
1942 {
1943   rgw_placement_rule dest_placement;
1944
1945   void encode(bufferlist& bl) const {
1946     ENCODE_START(1, 1, bl);
1947     encode(dest_placement, bl);
1948     ENCODE_FINISH(bl);
1949   }
1950
1951   void decode(bufferlist::const_iterator& bl) {
1952     DECODE_START(1, bl);
1953     decode(dest_placement, bl);
1954     DECODE_FINISH(bl);
1955   }
1956 };
1957 WRITE_CLASS_ENCODER(multipart_upload_info)
1958
1959 static inline void buf_to_hex(const unsigned char* const buf,
1960                               const size_t len,
1961                               char* const str)
1962 {
1963   str[0] = '\0';
1964   for (size_t i = 0; i < len; i++) {
1965     ::sprintf(&str[i*2], "%02x", static_cast<int>(buf[i]));
1966   }
1967 }
1968
1969 template<size_t N> static inline std::array<char, N * 2 + 1>
1970 buf_to_hex(const std::array<unsigned char, N>& buf)
1971 {
1972   static_assert(N > 0, "The input array must be at least one element long");
1973
1974   std::array<char, N * 2 + 1> hex_dest;
1975   buf_to_hex(buf.data(), N, hex_dest.data());
1976   return hex_dest;
1977 }
1978
1979 static inline int hexdigit(char c)
1980 {
1981   if (c >= '0' && c <= '9')
1982     return (c - '0');
1983   c = toupper(c);
1984   if (c >= 'A' && c <= 'F')
1985     return c - 'A' + 0xa;
1986   return -EINVAL;
1987 }
1988
1989 static inline int hex_to_buf(const char *hex, char *buf, int len)
1990 {
1991   int i = 0;
1992   const char *p = hex;
1993   while (*p) {
1994     if (i >= len)
1995       return -EINVAL;
1996     buf[i] = 0;
1997     int d = hexdigit(*p);
1998     if (d < 0)
1999       return d;
2000     buf[i] = d << 4;
2001     p++;
2002     if (!*p)
2003       return -EINVAL;
2004     d = hexdigit(*p);
2005     if (d < 0)
2006       return d;
2007     buf[i] += d;
2008     i++;
2009     p++;
2010   }
2011   return i;
2012 }
2013
2014 static inline int rgw_str_to_bool(const char *s, int def_val)
2015 {
2016   if (!s)
2017     return def_val;
2018
2019   return (strcasecmp(s, "true") == 0 ||
2020           strcasecmp(s, "on") == 0 ||
2021           strcasecmp(s, "yes") == 0 ||
2022           strcasecmp(s, "1") == 0);
2023 }
2024
2025 static inline void append_rand_alpha(CephContext *cct, const std::string& src, std::string& dest, int len)
2026 {
2027   dest = src;
2028   char buf[len + 1];
2029   gen_rand_alphanumeric(cct, buf, len);
2030   dest.append("_");
2031   dest.append(buf);
2032 }
2033
2034 static inline uint64_t rgw_rounded_kb(uint64_t bytes)
2035 {
2036   return (bytes + 1023) / 1024;
2037 }
2038
2039 static inline uint64_t rgw_rounded_objsize(uint64_t bytes)
2040 {
2041   return ((bytes + 4095) & ~4095);
2042 }
2043
2044 static inline uint64_t rgw_rounded_objsize_kb(uint64_t bytes)
2045 {
2046   return ((bytes + 4095) & ~4095) / 1024;
2047 }
2048
2049 /* implement combining step, S3 header canonicalization;  k is a
2050  * valid header and in lc form */
2051 void rgw_add_amz_meta_header(
2052   meta_map_t& x_meta_map,
2053   const std::string& k,
2054   const std::string& v);
2055
2056 enum rgw_set_action_if_set {
2057   DISCARD=0, OVERWRITE, APPEND
2058 };
2059
2060 bool rgw_set_amz_meta_header(
2061   meta_map_t& x_meta_map,
2062   const std::string& k,
2063   const std::string& v, rgw_set_action_if_set f);
2064
2065 extern std::string rgw_string_unquote(const std::string& s);
2066 extern void parse_csv_string(const std::string& ival, std::vector<std::string>& ovals);
2067 extern int parse_key_value(std::string& in_str, std::string& key, std::string& val);
2068 extern int parse_key_value(std::string& in_str, const char *delim, std::string& key, std::string& val);
2069
2070 extern boost::optional<std::pair<std::string_view,std::string_view>>
2071 parse_key_value(const std::string_view& in_str,
2072                 const std::string_view& delim);
2073 extern boost::optional<std::pair<std::string_view,std::string_view>>
2074 parse_key_value(const std::string_view& in_str);
2075
2076
2077 /** time parsing */
2078 extern int parse_time(const char *time_str, real_time *time);
2079 extern bool parse_rfc2616(const char *s, struct tm *t);
2080 extern bool parse_iso8601(const char *s, struct tm *t, uint32_t *pns = NULL, bool extended_format = true);
2081 extern std::string rgw_trim_whitespace(const std::string& src);
2082 extern std::string_view rgw_trim_whitespace(const std::string_view& src);
2083 extern std::string rgw_trim_quotes(const std::string& val);
2084
2085 extern void rgw_to_iso8601(const real_time& t, char *dest, int buf_size);
2086 extern void rgw_to_iso8601(const real_time& t, std::string *dest);
2087 extern std::string rgw_to_asctime(const utime_t& t);
2088
2089 struct perm_state_base {
2090   CephContext *cct;
2091   const rgw::IAM::Environment& env;
2092   rgw::auth::Identity *identity;
2093   const RGWBucketInfo bucket_info;
2094   int perm_mask;
2095   bool defer_to_bucket_acls;
2096   boost::optional<PublicAccessBlockConfiguration> bucket_access_conf;
2097
2098   perm_state_base(CephContext *_cct,
2099                   const rgw::IAM::Environment& _env,
2100                   rgw::auth::Identity *_identity,
2101                   const RGWBucketInfo& _bucket_info,
2102                   int _perm_mask,
2103                   bool _defer_to_bucket_acls,
2104                   boost::optional<PublicAccessBlockConfiguration> _bucket_acess_conf = boost::none) :
2105                                                 cct(_cct),
2106                                                 env(_env),
2107                                                 identity(_identity),
2108                                                 bucket_info(_bucket_info),
2109                                                 perm_mask(_perm_mask),
2110                                                 defer_to_bucket_acls(_defer_to_bucket_acls),
2111                                                 bucket_access_conf(_bucket_acess_conf)
2112   {}
2113
2114   virtual ~perm_state_base() {}
2115
2116   virtual const char *get_referer() const = 0;
2117   virtual std::optional<bool> get_request_payer() const = 0; /*
2118                                                               * empty state means that request_payer param was not passed in
2119                                                               */
2120
2121 };
2122
2123 struct perm_state : public perm_state_base {
2124   const char *referer;
2125   bool request_payer;
2126
2127   perm_state(CephContext *_cct,
2128              const rgw::IAM::Environment& _env,
2129              rgw::auth::Identity *_identity,
2130              const RGWBucketInfo& _bucket_info,
2131              int _perm_mask,
2132              bool _defer_to_bucket_acls,
2133              const char *_referer,
2134              bool _request_payer) : perm_state_base(_cct,
2135                                                     _env,
2136                                                     _identity,
2137                                                     _bucket_info,
2138                                                     _perm_mask,
2139                                                     _defer_to_bucket_acls),
2140                                     referer(_referer),
2141                                     request_payer(_request_payer) {}
2142
2143   const char *get_referer() const override {
2144     return referer;
2145   }
2146
2147   std::optional<bool> get_request_payer() const override {
2148     return request_payer;
2149   }
2150 };
2151
2152 /** Check if the req_state's user has the necessary permissions
2153  * to do the requested action  */
2154 bool verify_bucket_permission_no_policy(
2155   const DoutPrefixProvider* dpp,
2156   struct perm_state_base * const s,
2157   RGWAccessControlPolicy * const user_acl,
2158   RGWAccessControlPolicy * const bucket_acl,
2159   const int perm);
2160
2161 bool verify_user_permission_no_policy(const DoutPrefixProvider* dpp,
2162                                       struct perm_state_base * const s,
2163                                       RGWAccessControlPolicy * const user_acl,
2164                                       const int perm);
2165
2166 bool verify_object_permission_no_policy(const DoutPrefixProvider* dpp,
2167                                         struct perm_state_base * const s,
2168                                         RGWAccessControlPolicy * const user_acl,
2169                                         RGWAccessControlPolicy * const bucket_acl,
2170                                         RGWAccessControlPolicy * const object_acl,
2171                                         const int perm);
2172
2173 /** Check if the req_state's user has the necessary permissions
2174  * to do the requested action */
2175 rgw::IAM::Effect eval_identity_or_session_policies(const std::vector<rgw::IAM::Policy>& user_policies,
2176                           const rgw::IAM::Environment& env,
2177                           const uint64_t op,
2178                           const rgw::ARN& arn);
2179 bool verify_user_permission(const DoutPrefixProvider* dpp,
2180                             struct req_state * const s,
2181                             RGWAccessControlPolicy * const user_acl,
2182                             const std::vector<rgw::IAM::Policy>& user_policies,
2183                             const std::vector<rgw::IAM::Policy>& session_policies,
2184                             const rgw::ARN& res,
2185                             const uint64_t op);
2186 bool verify_user_permission_no_policy(const DoutPrefixProvider* dpp,
2187                                       struct req_state * const s,
2188                                       RGWAccessControlPolicy * const user_acl,
2189                                       const int perm);
2190 bool verify_user_permission(const DoutPrefixProvider* dpp,
2191                             struct req_state * const s,
2192                             const rgw::ARN& res,
2193                             const uint64_t op);
2194 bool verify_user_permission_no_policy(const DoutPrefixProvider* dpp,
2195                                       struct req_state * const s,
2196                                       int perm);
2197 bool verify_bucket_permission(
2198   const DoutPrefixProvider* dpp,
2199   struct req_state * const s,
2200   const rgw_bucket& bucket,
2201   RGWAccessControlPolicy * const user_acl,
2202   RGWAccessControlPolicy * const bucket_acl,
2203   const boost::optional<rgw::IAM::Policy>& bucket_policy,
2204   const std::vector<rgw::IAM::Policy>& identity_policies,
2205   const std::vector<rgw::IAM::Policy>& session_policies,
2206   const uint64_t op);
2207 bool verify_bucket_permission(const DoutPrefixProvider* dpp, struct req_state * const s, const uint64_t op);
2208 bool verify_bucket_permission_no_policy(
2209   const DoutPrefixProvider* dpp,
2210   struct req_state * const s,
2211   RGWAccessControlPolicy * const user_acl,
2212   RGWAccessControlPolicy * const bucket_acl,
2213   const int perm);
2214 bool verify_bucket_permission_no_policy(const DoutPrefixProvider* dpp,
2215                                         struct req_state * const s,
2216                                         const int perm);
2217 int verify_bucket_owner_or_policy(struct req_state* const s,
2218                                   const uint64_t op);
2219 extern bool verify_object_permission(
2220   const DoutPrefixProvider* dpp,
2221   struct req_state * const s,
2222   const rgw_obj& obj,
2223   RGWAccessControlPolicy * const user_acl,
2224   RGWAccessControlPolicy * const bucket_acl,
2225   RGWAccessControlPolicy * const object_acl,
2226   const boost::optional<rgw::IAM::Policy>& bucket_policy,
2227   const std::vector<rgw::IAM::Policy>& identity_policies,
2228   const std::vector<rgw::IAM::Policy>& session_policies,
2229   const uint64_t op);
2230 extern bool verify_object_permission(const DoutPrefixProvider* dpp, struct req_state *s, uint64_t op);
2231 extern bool verify_object_permission_no_policy(
2232   const DoutPrefixProvider* dpp,
2233   struct req_state * const s,
2234   RGWAccessControlPolicy * const user_acl,
2235   RGWAccessControlPolicy * const bucket_acl,
2236   RGWAccessControlPolicy * const object_acl,
2237   int perm);
2238 extern bool verify_object_permission_no_policy(const DoutPrefixProvider* dpp, struct req_state *s,
2239                                                int perm);
2240 extern int verify_object_lock(
2241   const DoutPrefixProvider* dpp,
2242   const rgw::sal::Attrs& attrs,
2243   const bool bypass_perm,
2244   const bool bypass_governance_mode);
2245
2246 /** Convert an input URL into a sane object name
2247  * by converting %-escaped std::strings into characters, etc*/
2248 extern void rgw_uri_escape_char(char c, std::string& dst);
2249 extern std::string url_decode(const std::string_view& src_str,
2250                               bool in_query = false);
2251 extern void url_encode(const std::string& src, std::string& dst,
2252                        bool encode_slash = true);
2253 extern std::string url_encode(const std::string& src, bool encode_slash = true);
2254 extern std::string url_remove_prefix(const std::string& url); // Removes hhtp, https and www from url
2255 /* destination should be CEPH_CRYPTO_HMACSHA1_DIGESTSIZE bytes long */
2256 extern void calc_hmac_sha1(const char *key, int key_len,
2257                           const char *msg, int msg_len, char *dest);
2258
2259 static inline sha1_digest_t
2260 calc_hmac_sha1(const std::string_view& key, const std::string_view& msg) {
2261   sha1_digest_t dest;
2262   calc_hmac_sha1(key.data(), key.size(), msg.data(), msg.size(),
2263                  reinterpret_cast<char*>(dest.v));
2264   return dest;
2265 }
2266
2267 /* destination should be CEPH_CRYPTO_HMACSHA256_DIGESTSIZE bytes long */
2268 extern void calc_hmac_sha256(const char *key, int key_len,
2269                              const char *msg, int msg_len,
2270                              char *dest);
2271
2272 static inline sha256_digest_t
2273 calc_hmac_sha256(const char *key, const int key_len,
2274                  const char *msg, const int msg_len) {
2275   sha256_digest_t dest;
2276   calc_hmac_sha256(key, key_len, msg, msg_len,
2277                    reinterpret_cast<char*>(dest.v));
2278   return dest;
2279 }
2280
2281 static inline sha256_digest_t
2282 calc_hmac_sha256(const std::string_view& key, const std::string_view& msg) {
2283   sha256_digest_t dest;
2284   calc_hmac_sha256(key.data(), key.size(),
2285                    msg.data(), msg.size(),
2286                    reinterpret_cast<char*>(dest.v));
2287   return dest;
2288 }
2289
2290 static inline sha256_digest_t
2291 calc_hmac_sha256(const sha256_digest_t &key,
2292                  const std::string_view& msg) {
2293   sha256_digest_t dest;
2294   calc_hmac_sha256(reinterpret_cast<const char*>(key.v), sha256_digest_t::SIZE,
2295                    msg.data(), msg.size(),
2296                    reinterpret_cast<char*>(dest.v));
2297   return dest;
2298 }
2299
2300 static inline sha256_digest_t
2301 calc_hmac_sha256(const std::vector<unsigned char>& key,
2302                  const std::string_view& msg) {
2303   sha256_digest_t dest;
2304   calc_hmac_sha256(reinterpret_cast<const char*>(key.data()), key.size(),
2305                    msg.data(), msg.size(),
2306                    reinterpret_cast<char*>(dest.v));
2307   return dest;
2308 }
2309
2310 template<size_t KeyLenN>
2311 static inline sha256_digest_t
2312 calc_hmac_sha256(const std::array<unsigned char, KeyLenN>& key,
2313                  const std::string_view& msg) {
2314   sha256_digest_t dest;
2315   calc_hmac_sha256(reinterpret_cast<const char*>(key.data()), key.size(),
2316                    msg.data(), msg.size(),
2317                    reinterpret_cast<char*>(dest.v));
2318   return dest;
2319 }
2320
2321 extern sha256_digest_t calc_hash_sha256(const std::string_view& msg);
2322
2323 extern ceph::crypto::SHA256* calc_hash_sha256_open_stream();
2324 extern void calc_hash_sha256_update_stream(ceph::crypto::SHA256* hash,
2325                                            const char* msg,
2326                                            int len);
2327 extern std::string calc_hash_sha256_close_stream(ceph::crypto::SHA256** phash);
2328 extern std::string calc_hash_sha256_restart_stream(ceph::crypto::SHA256** phash);
2329
2330 extern int rgw_parse_op_type_list(const std::string& str, uint32_t *perm);
2331
2332 static constexpr uint32_t MATCH_POLICY_ACTION = 0x01;
2333 static constexpr uint32_t MATCH_POLICY_RESOURCE = 0x02;
2334 static constexpr uint32_t MATCH_POLICY_ARN = 0x04;
2335 static constexpr uint32_t MATCH_POLICY_STRING = 0x08;
2336
2337 extern bool match_policy(std::string_view pattern, std::string_view input,
2338                          uint32_t flag);
2339
2340 extern std::string camelcase_dash_http_attr(const std::string& orig);
2341 extern std::string lowercase_dash_http_attr(const std::string& orig);
2342
2343 void rgw_setup_saved_curl_handles();
2344 void rgw_release_all_curl_handles();
2345
2346 static inline void rgw_escape_str(const std::string& s, char esc_char,
2347                                   char special_char, std::string *dest)
2348 {
2349   const char *src = s.c_str();
2350   char dest_buf[s.size() * 2 + 1];
2351   char *destp = dest_buf;
2352
2353   for (size_t i = 0; i < s.size(); i++) {
2354     char c = src[i];
2355     if (c == esc_char || c == special_char) {
2356       *destp++ = esc_char;
2357     }
2358     *destp++ = c;
2359   }
2360   *destp++ = '\0';
2361   *dest = dest_buf;
2362 }
2363
2364 static inline ssize_t rgw_unescape_str(const std::string& s, ssize_t ofs,
2365                                        char esc_char, char special_char,
2366                                        std::string *dest)
2367 {
2368   const char *src = s.c_str();
2369   char dest_buf[s.size() + 1];
2370   char *destp = dest_buf;
2371   bool esc = false;
2372
2373   dest_buf[0] = '\0';
2374
2375   for (size_t i = ofs; i < s.size(); i++) {
2376     char c = src[i];
2377     if (!esc && c == esc_char) {
2378       esc = true;
2379       continue;
2380     }
2381     if (!esc && c == special_char) {
2382       *destp = '\0';
2383       *dest = dest_buf;
2384       return (ssize_t)i + 1;
2385     }
2386     *destp++ = c;
2387     esc = false;
2388   }
2389   *destp = '\0';
2390   *dest = dest_buf;
2391   return std::string::npos;
2392 }
2393
2394 static inline std::string rgw_bl_str(ceph::buffer::list& raw)
2395 {
2396   size_t len = raw.length();
2397   std::string s(raw.c_str(), len);
2398   while (len && !s[len - 1]) {
2399     --len;
2400     s.resize(len);
2401   }
2402   return s;
2403 }
2404
2405 template <typename T>
2406 int decode_bl(bufferlist& bl, T& t)
2407 {
2408   auto iter = bl.cbegin();
2409   try {
2410     decode(t, iter);
2411   } catch (buffer::error& err) {
2412     return -EIO;
2413   }
2414   return 0;
2415 }