ceph/src/rgw/rgw_common.h

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab ft=cpp
   3
   4 /*
   5  * Ceph - scalable distributed file system
   6  *
   7  * Copyright (C) 2004-2009 Sage Weil <sage@newdream.net>
   8  * Copyright (C) 2015 Yehuda Sadeh <yehuda@redhat.com>
   9  *
  10  * This is free software; you can redistribute it and/or
  11  * modify it under the terms of the GNU Lesser General Public
  12  * License version 2.1, as published by the Free Software
  13  * Foundation.  See file COPYING.
  14  *
  15  */
  16
  17 #pragma once
  18
  19 #include <array>
  20 #include <string_view>
  21 #include <atomic>
  22 #include <unordered_map>
  23
  24 #include "common/ceph_crypto.h"
  25 #include "common/random_string.h"
  26 #include "rgw_acl.h"
  27 #include "rgw_bucket_layout.h"
  28 #include "rgw_cors.h"
  29 #include "rgw_iam_policy.h"
  30 #include "rgw_quota.h"
  31 #include "rgw_string.h"
  32 #include "common/async/yield_context.h"
  33 #include "rgw_website.h"
  34 #include "rgw_object_lock.h"
  35 #include "rgw_tag.h"
  36 #include "rgw_op_type.h"
  37 #include "rgw_sync_policy.h"
  38 #include "cls/version/cls_version_types.h"
  39 #include "cls/user/cls_user_types.h"
  40 #include "cls/rgw/cls_rgw_types.h"
  41 #include "include/rados/librados.hpp"
  42 #include "rgw_public_access.h"
  43 #include "common/tracer.h"
  44
  45 namespace ceph {
  46   class Formatter;
  47 }
  48
  49 namespace rgw::sal {
  50   class User;
  51   class Bucket;
  52   class Object;
  53   using Attrs = std::map<std::string, ceph::buffer::list>;
  54 }
  55
  56 using ceph::crypto::MD5;
  57
  58 #define RGW_ATTR_PREFIX  "user.rgw."
  59
  60 #define RGW_HTTP_RGWX_ATTR_PREFIX "RGWX_ATTR_"
  61 #define RGW_HTTP_RGWX_ATTR_PREFIX_OUT "Rgwx-Attr-"
  62
  63 #define RGW_AMZ_PREFIX "x-amz-"
  64 #define RGW_AMZ_META_PREFIX RGW_AMZ_PREFIX "meta-"
  65 #define RGW_AMZ_WEBSITE_REDIRECT_LOCATION RGW_AMZ_PREFIX "website-redirect-location"
  66 #define RGW_AMZ_TAG_COUNT RGW_AMZ_PREFIX "tagging-count"
  67
  68 #define RGW_SYS_PARAM_PREFIX "rgwx-"
  69
  70 #define RGW_ATTR_ACL            RGW_ATTR_PREFIX "acl"
  71 #define RGW_ATTR_RATELIMIT              RGW_ATTR_PREFIX "ratelimit"
  72 #define RGW_ATTR_LC            RGW_ATTR_PREFIX "lc"
  73 #define RGW_ATTR_CORS           RGW_ATTR_PREFIX "cors"
  74 #define RGW_ATTR_ETAG           RGW_ATTR_PREFIX "etag"
  75 #define RGW_ATTR_BUCKETS        RGW_ATTR_PREFIX "buckets"
  76 #define RGW_ATTR_META_PREFIX    RGW_ATTR_PREFIX RGW_AMZ_META_PREFIX
  77 #define RGW_ATTR_CONTENT_TYPE   RGW_ATTR_PREFIX "content_type"
  78 #define RGW_ATTR_CACHE_CONTROL  RGW_ATTR_PREFIX "cache_control"
  79 #define RGW_ATTR_CONTENT_DISP   RGW_ATTR_PREFIX "content_disposition"
  80 #define RGW_ATTR_CONTENT_ENC    RGW_ATTR_PREFIX "content_encoding"
  81 #define RGW_ATTR_CONTENT_LANG   RGW_ATTR_PREFIX "content_language"
  82 #define RGW_ATTR_EXPIRES        RGW_ATTR_PREFIX "expires"
  83 #define RGW_ATTR_DELETE_AT      RGW_ATTR_PREFIX "delete_at"
  84 #define RGW_ATTR_ID_TAG         RGW_ATTR_PREFIX "idtag"
  85 #define RGW_ATTR_TAIL_TAG       RGW_ATTR_PREFIX "tail_tag"
  86 #define RGW_ATTR_SHADOW_OBJ     RGW_ATTR_PREFIX "shadow_name"
  87 #define RGW_ATTR_MANIFEST       RGW_ATTR_PREFIX "manifest"
  88 #define RGW_ATTR_USER_MANIFEST  RGW_ATTR_PREFIX "user_manifest"
  89 #define RGW_ATTR_AMZ_WEBSITE_REDIRECT_LOCATION  RGW_ATTR_PREFIX RGW_AMZ_WEBSITE_REDIRECT_LOCATION
  90 #define RGW_ATTR_SLO_MANIFEST   RGW_ATTR_PREFIX "slo_manifest"
  91 /* Information whether an object is SLO or not must be exposed to
  92  * user through custom HTTP header named X-Static-Large-Object. */
  93 #define RGW_ATTR_SLO_UINDICATOR RGW_ATTR_META_PREFIX "static-large-object"
  94 #define RGW_ATTR_X_ROBOTS_TAG   RGW_ATTR_PREFIX "x-robots-tag"
  95 #define RGW_ATTR_STORAGE_CLASS  RGW_ATTR_PREFIX "storage_class"
  96
  97 /* S3 Object Lock*/
  98 #define RGW_ATTR_OBJECT_LOCK        RGW_ATTR_PREFIX "object-lock"
  99 #define RGW_ATTR_OBJECT_RETENTION   RGW_ATTR_PREFIX "object-retention"
 100 #define RGW_ATTR_OBJECT_LEGAL_HOLD  RGW_ATTR_PREFIX "object-legal-hold"
 101
 102
 103 #define RGW_ATTR_PG_VER         RGW_ATTR_PREFIX "pg_ver"
 104 #define RGW_ATTR_SOURCE_ZONE    RGW_ATTR_PREFIX "source_zone"
 105 #define RGW_ATTR_TAGS           RGW_ATTR_PREFIX RGW_AMZ_PREFIX "tagging"
 106
 107 #define RGW_ATTR_TEMPURL_KEY1   RGW_ATTR_META_PREFIX "temp-url-key"
 108 #define RGW_ATTR_TEMPURL_KEY2   RGW_ATTR_META_PREFIX "temp-url-key-2"
 109
 110 /* Account/container quota of the Swift API. */
 111 #define RGW_ATTR_QUOTA_NOBJS    RGW_ATTR_META_PREFIX "quota-count"
 112 #define RGW_ATTR_QUOTA_MSIZE    RGW_ATTR_META_PREFIX "quota-bytes"
 113
 114 /* Static Web Site of Swift API. */
 115 #define RGW_ATTR_WEB_INDEX      RGW_ATTR_META_PREFIX "web-index"
 116 #define RGW_ATTR_WEB_ERROR      RGW_ATTR_META_PREFIX "web-error"
 117 #define RGW_ATTR_WEB_LISTINGS   RGW_ATTR_META_PREFIX "web-listings"
 118 #define RGW_ATTR_WEB_LIST_CSS   RGW_ATTR_META_PREFIX "web-listings-css"
 119 #define RGW_ATTR_SUBDIR_MARKER  RGW_ATTR_META_PREFIX "web-directory-type"
 120
 121 #define RGW_ATTR_OLH_PREFIX     RGW_ATTR_PREFIX "olh."
 122
 123 #define RGW_ATTR_OLH_INFO       RGW_ATTR_OLH_PREFIX "info"
 124 #define RGW_ATTR_OLH_VER        RGW_ATTR_OLH_PREFIX "ver"
 125 #define RGW_ATTR_OLH_ID_TAG     RGW_ATTR_OLH_PREFIX "idtag"
 126 #define RGW_ATTR_OLH_PENDING_PREFIX RGW_ATTR_OLH_PREFIX "pending."
 127
 128 #define RGW_ATTR_COMPRESSION    RGW_ATTR_PREFIX "compression"
 129
 130 #define RGW_ATTR_APPEND_PART_NUM    RGW_ATTR_PREFIX "append_part_num"
 131
 132 /* IAM Policy */
 133 #define RGW_ATTR_IAM_POLICY     RGW_ATTR_PREFIX "iam-policy"
 134 #define RGW_ATTR_USER_POLICY    RGW_ATTR_PREFIX "user-policy"
 135 #define RGW_ATTR_PUBLIC_ACCESS  RGW_ATTR_PREFIX "public-access"
 136
 137 /* RGW File Attributes */
 138 #define RGW_ATTR_UNIX_KEY1      RGW_ATTR_PREFIX "unix-key1"
 139 #define RGW_ATTR_UNIX1          RGW_ATTR_PREFIX "unix1"
 140
 141 #define RGW_ATTR_CRYPT_PREFIX   RGW_ATTR_PREFIX "crypt."
 142 #define RGW_ATTR_CRYPT_MODE     RGW_ATTR_CRYPT_PREFIX "mode"
 143 #define RGW_ATTR_CRYPT_KEYMD5   RGW_ATTR_CRYPT_PREFIX "keymd5"
 144 #define RGW_ATTR_CRYPT_KEYID    RGW_ATTR_CRYPT_PREFIX "keyid"
 145 #define RGW_ATTR_CRYPT_KEYSEL   RGW_ATTR_CRYPT_PREFIX "keysel"
 146 #define RGW_ATTR_CRYPT_CONTEXT  RGW_ATTR_CRYPT_PREFIX "context"
 147 #define RGW_ATTR_CRYPT_DATAKEY  RGW_ATTR_CRYPT_PREFIX "datakey"
 148
 149 /* SSE-S3 Encryption Attributes */
 150 #define RGW_ATTR_BUCKET_ENCRYPTION_PREFIX RGW_ATTR_PREFIX "sse-s3."
 151 #define RGW_ATTR_BUCKET_ENCRYPTION_POLICY RGW_ATTR_BUCKET_ENCRYPTION_PREFIX "policy"
 152 #define RGW_ATTR_BUCKET_ENCRYPTION_KEY_ID RGW_ATTR_BUCKET_ENCRYPTION_PREFIX "key-id"
 153
 154 #define RGW_ATTR_TRACE RGW_ATTR_PREFIX "trace"
 155
 156 #define RGW_FORMAT_PLAIN        0
 157 #define RGW_FORMAT_XML          1
 158 #define RGW_FORMAT_JSON         2
 159 #define RGW_FORMAT_HTML         3
 160
 161 #define RGW_CAP_READ            0x1
 162 #define RGW_CAP_WRITE           0x2
 163 #define RGW_CAP_ALL             (RGW_CAP_READ | RGW_CAP_WRITE)
 164
 165 #define RGW_REST_SWIFT          0x1
 166 #define RGW_REST_SWIFT_AUTH     0x2
 167 #define RGW_REST_S3             0x4
 168 #define RGW_REST_WEBSITE     0x8
 169 #define RGW_REST_STS            0x10
 170 #define RGW_REST_IAM            0x20
 171
 172 #define RGW_SUSPENDED_USER_AUID (uint64_t)-2
 173
 174 #define RGW_OP_TYPE_READ         0x01
 175 #define RGW_OP_TYPE_WRITE        0x02
 176 #define RGW_OP_TYPE_DELETE       0x04
 177
 178 #define RGW_OP_TYPE_MODIFY       (RGW_OP_TYPE_WRITE | RGW_OP_TYPE_DELETE)
 179 #define RGW_OP_TYPE_ALL          (RGW_OP_TYPE_READ | RGW_OP_TYPE_WRITE | RGW_OP_TYPE_DELETE)
 180
 181 #define RGW_DEFAULT_MAX_BUCKETS 1000
 182
 183 #define RGW_DEFER_TO_BUCKET_ACLS_RECURSE 1
 184 #define RGW_DEFER_TO_BUCKET_ACLS_FULL_CONTROL 2
 185
 186 #define STATUS_CREATED           1900
 187 #define STATUS_ACCEPTED          1901
 188 #define STATUS_NO_CONTENT        1902
 189 #define STATUS_PARTIAL_CONTENT   1903
 190 #define STATUS_REDIRECT          1904
 191 #define STATUS_NO_APPLY          1905
 192 #define STATUS_APPLIED           1906
 193
 194 #define ERR_INVALID_BUCKET_NAME  2000
 195 #define ERR_INVALID_OBJECT_NAME  2001
 196 #define ERR_NO_SUCH_BUCKET       2002
 197 #define ERR_METHOD_NOT_ALLOWED   2003
 198 #define ERR_INVALID_DIGEST       2004
 199 #define ERR_BAD_DIGEST           2005
 200 #define ERR_UNRESOLVABLE_EMAIL   2006
 201 #define ERR_INVALID_PART         2007
 202 #define ERR_INVALID_PART_ORDER   2008
 203 #define ERR_NO_SUCH_UPLOAD       2009
 204 #define ERR_REQUEST_TIMEOUT      2010
 205 #define ERR_LENGTH_REQUIRED      2011
 206 #define ERR_REQUEST_TIME_SKEWED  2012
 207 #define ERR_BUCKET_EXISTS        2013
 208 #define ERR_BAD_URL              2014
 209 #define ERR_PRECONDITION_FAILED  2015
 210 #define ERR_NOT_MODIFIED         2016
 211 #define ERR_INVALID_UTF8         2017
 212 #define ERR_UNPROCESSABLE_ENTITY 2018
 213 #define ERR_TOO_LARGE            2019
 214 #define ERR_TOO_MANY_BUCKETS     2020
 215 #define ERR_INVALID_REQUEST      2021
 216 #define ERR_TOO_SMALL            2022
 217 #define ERR_NOT_FOUND            2023
 218 #define ERR_PERMANENT_REDIRECT   2024
 219 #define ERR_LOCKED               2025
 220 #define ERR_QUOTA_EXCEEDED       2026
 221 #define ERR_SIGNATURE_NO_MATCH   2027
 222 #define ERR_INVALID_ACCESS_KEY   2028
 223 #define ERR_MALFORMED_XML        2029
 224 #define ERR_USER_EXIST           2030
 225 #define ERR_NOT_SLO_MANIFEST     2031
 226 #define ERR_EMAIL_EXIST          2032
 227 #define ERR_KEY_EXIST            2033
 228 #define ERR_INVALID_SECRET_KEY   2034
 229 #define ERR_INVALID_KEY_TYPE     2035
 230 #define ERR_INVALID_CAP          2036
 231 #define ERR_INVALID_TENANT_NAME  2037
 232 #define ERR_WEBSITE_REDIRECT     2038
 233 #define ERR_NO_SUCH_WEBSITE_CONFIGURATION 2039
 234 #define ERR_AMZ_CONTENT_SHA256_MISMATCH 2040
 235 #define ERR_NO_SUCH_LC           2041
 236 #define ERR_NO_SUCH_USER         2042
 237 #define ERR_NO_SUCH_SUBUSER      2043
 238 #define ERR_MFA_REQUIRED         2044
 239 #define ERR_NO_SUCH_CORS_CONFIGURATION 2045
 240 #define ERR_NO_SUCH_OBJECT_LOCK_CONFIGURATION  2046
 241 #define ERR_INVALID_RETENTION_PERIOD 2047
 242 #define ERR_NO_SUCH_BUCKET_ENCRYPTION_CONFIGURATION 2048
 243 #define ERR_USER_SUSPENDED       2100
 244 #define ERR_INTERNAL_ERROR       2200
 245 #define ERR_NOT_IMPLEMENTED      2201
 246 #define ERR_SERVICE_UNAVAILABLE  2202
 247 #define ERR_ROLE_EXISTS          2203
 248 #define ERR_MALFORMED_DOC        2204
 249 #define ERR_NO_ROLE_FOUND        2205
 250 #define ERR_DELETE_CONFLICT      2206
 251 #define ERR_NO_SUCH_BUCKET_POLICY  2207
 252 #define ERR_INVALID_LOCATION_CONSTRAINT 2208
 253 #define ERR_TAG_CONFLICT         2209
 254 #define ERR_INVALID_TAG          2210
 255 #define ERR_ZERO_IN_URL          2211
 256 #define ERR_MALFORMED_ACL_ERROR  2212
 257 #define ERR_ZONEGROUP_DEFAULT_PLACEMENT_MISCONFIGURATION 2213
 258 #define ERR_INVALID_ENCRYPTION_ALGORITHM                 2214
 259 #define ERR_INVALID_CORS_RULES_ERROR                     2215
 260 #define ERR_NO_CORS_FOUND        2216
 261 #define ERR_INVALID_WEBSITE_ROUTING_RULES_ERROR          2217
 262 #define ERR_RATE_LIMITED         2218
 263 #define ERR_POSITION_NOT_EQUAL_TO_LENGTH                 2219
 264 #define ERR_OBJECT_NOT_APPENDABLE                        2220
 265 #define ERR_INVALID_BUCKET_STATE                         2221
 266 #define ERR_INVALID_OBJECT_STATE                         2222
 267
 268 #define ERR_BUSY_RESHARDING      2300
 269 #define ERR_NO_SUCH_ENTITY       2301
 270 #define ERR_LIMIT_EXCEEDED       2302
 271
 272 // STS Errors
 273 #define ERR_PACKED_POLICY_TOO_LARGE 2400
 274 #define ERR_INVALID_IDENTITY_TOKEN  2401
 275
 276 #define ERR_NO_SUCH_TAG_SET 2402
 277
 278 #ifndef UINT32_MAX
 279 #define UINT32_MAX (0xffffffffu)
 280 #endif
 281
 282 struct req_state;
 283
 284 typedef void *RGWAccessHandle;
 285
 286 enum RGWIntentEvent {
 287   DEL_OBJ = 0,
 288   DEL_DIR = 1,
 289 };
 290
 291 enum HostStyle {
 292   PathStyle = 0,
 293   VirtualStyle = 1,
 294 };
 295
 296 /** Store error returns for output at a different point in the program */
 297 struct rgw_err {
 298   rgw_err();
 299   void clear();
 300   bool is_clear() const;
 301   bool is_err() const;
 302   friend std::ostream& operator<<(std::ostream& oss, const rgw_err &err);
 303
 304   int http_ret;
 305   int ret;
 306   std::string err_code;
 307   std::string message;
 308 };
 309
 310
 311 /* Helper class used for RGWHTTPArgs parsing */
 312 class NameVal
 313 {
 314    const std::string str;
 315    std::string name;
 316    std::string val;
 317  public:
 318     explicit NameVal(const std::string& nv) : str(nv) {}
 319
 320     int parse();
 321
 322     std::string& get_name() { return name; }
 323     std::string& get_val() { return val; }
 324 };
 325
 326 /** Stores the XML arguments associated with the HTTP request in req_state*/
 327 class RGWHTTPArgs {
 328   std::string str, empty_str;
 329   std::map<std::string, std::string> val_map;
 330   std::map<std::string, std::string> sys_val_map;
 331   std::map<std::string, std::string> sub_resources;
 332   bool has_resp_modifier = false;
 333   bool admin_subresource_added = false;
 334  public:
 335   RGWHTTPArgs() = default;
 336   explicit RGWHTTPArgs(const std::string& s, const DoutPrefixProvider *dpp) {
 337       set(s);
 338       parse(dpp);
 339   }
 340
 341   /** Set the arguments; as received */
 342   void set(const std::string& s) {
 343     has_resp_modifier = false;
 344     val_map.clear();
 345     sub_resources.clear();
 346     str = s;
 347   }
 348   /** parse the received arguments */
 349   int parse(const DoutPrefixProvider *dpp);
 350   void append(const std::string& name, const std::string& val);
 351   void remove(const std::string& name);
 352   /** Get the value for a specific argument parameter */
 353   const std::string& get(const std::string& name, bool *exists = NULL) const;
 354   boost::optional<const std::string&>
 355   get_optional(const std::string& name) const;
 356   int get_bool(const std::string& name, bool *val, bool *exists) const;
 357   int get_bool(const char *name, bool *val, bool *exists) const;
 358   void get_bool(const char *name, bool *val, bool def_val) const;
 359   int get_int(const char *name, int *val, int def_val) const;
 360
 361   /** Get the value for specific system argument parameter */
 362   std::string sys_get(const std::string& name, bool *exists = nullptr) const;
 363
 364   /** see if a parameter is contained in this RGWHTTPArgs */
 365   bool exists(const char *name) const {
 366     return (val_map.find(name) != std::end(val_map));
 367   }
 368   bool sub_resource_exists(const char *name) const {
 369     return (sub_resources.find(name) != std::end(sub_resources));
 370   }
 371   bool exist_obj_excl_sub_resource() const {
 372     const char* const obj_sub_resource[] = {"append", "torrent", "uploadId",
 373                                             "partNumber", "versionId"};
 374     for (unsigned i = 0; i != std::size(obj_sub_resource); i++) {
 375       if (sub_resource_exists(obj_sub_resource[i])) return true;
 376     }
 377     return false;
 378   }
 379
 380   std::map<std::string, std::string>& get_params() {
 381     return val_map;
 382   }
 383   const std::map<std::string, std::string>& get_params() const {
 384     return val_map;
 385   }
 386   std::map<std::string, std::string>& get_sys_params() {
 387     return sys_val_map;
 388   }
 389   const std::map<std::string, std::string>& get_sys_params() const {
 390     return sys_val_map;
 391   }
 392   const std::map<std::string, std::string>& get_sub_resources() const {
 393     return sub_resources;
 394   }
 395   unsigned get_num_params() const {
 396     return val_map.size();
 397   }
 398   bool has_response_modifier() const {
 399     return has_resp_modifier;
 400   }
 401   void set_system() { /* make all system params visible */
 402     std::map<std::string, std::string>::iterator iter;
 403     for (iter = sys_val_map.begin(); iter != sys_val_map.end(); ++iter) {
 404       val_map[iter->first] = iter->second;
 405     }
 406   }
 407   const std::string& get_str() {
 408     return str;
 409   }
 410 }; // RGWHTTPArgs
 411
 412 const char *rgw_conf_get(const std::map<std::string, std::string, ltstr_nocase>& conf_map, const char *name, const char *def_val);
 413 int rgw_conf_get_int(const std::map<std::string, std::string, ltstr_nocase>& conf_map, const char *name, int def_val);
 414 bool rgw_conf_get_bool(const std::map<std::string, std::string, ltstr_nocase>& conf_map, const char *name, bool def_val);
 415
 416 class RGWEnv;
 417
 418 class RGWConf {
 419   friend class RGWEnv;
 420   int enable_ops_log;
 421   int enable_usage_log;
 422   uint8_t defer_to_bucket_acls;
 423   void init(CephContext *cct);
 424 public:
 425   RGWConf()
 426     : enable_ops_log(1),
 427       enable_usage_log(1),
 428       defer_to_bucket_acls(0) {
 429   }
 430 };
 431
 432 class RGWEnv {
 433   std::map<std::string, std::string, ltstr_nocase> env_map;
 434   RGWConf conf;
 435 public:
 436   void init(CephContext *cct);
 437   void init(CephContext *cct, char **envp);
 438   void set(std::string name, std::string val);
 439   const char *get(const char *name, const char *def_val = nullptr) const;
 440   int get_int(const char *name, int def_val = 0) const;
 441   bool get_bool(const char *name, bool def_val = 0);
 442   size_t get_size(const char *name, size_t def_val = 0) const;
 443   bool exists(const char *name) const;
 444   bool exists_prefix(const char *prefix) const;
 445   void remove(const char *name);
 446   const std::map<std::string, std::string, ltstr_nocase>& get_map() const { return env_map; }
 447   int get_enable_ops_log() const {
 448     return conf.enable_ops_log;
 449   }
 450
 451   int get_enable_usage_log() const {
 452     return conf.enable_usage_log;
 453   }
 454
 455   int get_defer_to_bucket_acls() const {
 456     return conf.defer_to_bucket_acls;
 457   }
 458 };
 459
 460 // return true if the connection is secure. this either means that the
 461 // connection arrived via ssl, or was forwarded as https by a trusted proxy
 462 bool rgw_transport_is_secure(CephContext *cct, const RGWEnv& env);
 463
 464 enum http_op {
 465   OP_GET,
 466   OP_PUT,
 467   OP_DELETE,
 468   OP_HEAD,
 469   OP_POST,
 470   OP_COPY,
 471   OP_OPTIONS,
 472   OP_UNKNOWN,
 473 };
 474
 475 class RGWAccessControlPolicy;
 476 class JSONObj;
 477
 478 struct RGWAccessKey {
 479   std::string id; // AccessKey
 480   std::string key; // SecretKey
 481   std::string subuser;
 482
 483   RGWAccessKey() {}
 484   RGWAccessKey(std::string _id, std::string _key)
 485     : id(std::move(_id)), key(std::move(_key)) {}
 486
 487   void encode(bufferlist& bl) const {
 488     ENCODE_START(2, 2, bl);
 489     encode(id, bl);
 490     encode(key, bl);
 491     encode(subuser, bl);
 492     ENCODE_FINISH(bl);
 493   }
 494
 495   void decode(bufferlist::const_iterator& bl) {
 496      DECODE_START_LEGACY_COMPAT_LEN_32(2, 2, 2, bl);
 497      decode(id, bl);
 498      decode(key, bl);
 499      decode(subuser, bl);
 500      DECODE_FINISH(bl);
 501   }
 502   void dump(Formatter *f) const;
 503   void dump_plain(Formatter *f) const;
 504   void dump(Formatter *f, const std::string& user, bool swift) const;
 505   static void generate_test_instances(std::list<RGWAccessKey*>& o);
 506
 507   void decode_json(JSONObj *obj);
 508   void decode_json(JSONObj *obj, bool swift);
 509 };
 510 WRITE_CLASS_ENCODER(RGWAccessKey)
 511
 512 struct RGWSubUser {
 513   std::string name;
 514   uint32_t perm_mask;
 515
 516   RGWSubUser() : perm_mask(0) {}
 517   void encode(bufferlist& bl) const {
 518     ENCODE_START(2, 2, bl);
 519     encode(name, bl);
 520     encode(perm_mask, bl);
 521     ENCODE_FINISH(bl);
 522   }
 523
 524   void decode(bufferlist::const_iterator& bl) {
 525      DECODE_START_LEGACY_COMPAT_LEN_32(2, 2, 2, bl);
 526      decode(name, bl);
 527      decode(perm_mask, bl);
 528      DECODE_FINISH(bl);
 529   }
 530   void dump(Formatter *f) const;
 531   void dump(Formatter *f, const std::string& user) const;
 532   static void generate_test_instances(std::list<RGWSubUser*>& o);
 533
 534   void decode_json(JSONObj *obj);
 535 };
 536 WRITE_CLASS_ENCODER(RGWSubUser)
 537
 538 class RGWUserCaps
 539 {
 540   std::map<std::string, uint32_t> caps;
 541
 542   int get_cap(const std::string& cap, std::string& type, uint32_t *perm);
 543   int add_cap(const std::string& cap);
 544   int remove_cap(const std::string& cap);
 545 public:
 546   static int parse_cap_perm(const std::string& str, uint32_t *perm);
 547   int add_from_string(const std::string& str);
 548   int remove_from_string(const std::string& str);
 549
 550   void encode(bufferlist& bl) const {
 551      ENCODE_START(1, 1, bl);
 552      encode(caps, bl);
 553      ENCODE_FINISH(bl);
 554   }
 555   void decode(bufferlist::const_iterator& bl) {
 556      DECODE_START(1, bl);
 557      decode(caps, bl);
 558      DECODE_FINISH(bl);
 559   }
 560   int check_cap(const std::string& cap, uint32_t perm) const;
 561   bool is_valid_cap_type(const std::string& tp);
 562   void dump(Formatter *f) const;
 563   void dump(Formatter *f, const char *name) const;
 564
 565   void decode_json(JSONObj *obj);
 566 };
 567 WRITE_CLASS_ENCODER(RGWUserCaps)
 568
 569 void encode_json(const char *name, const obj_version& v, Formatter *f);
 570 void encode_json(const char *name, const RGWUserCaps& val, Formatter *f);
 571
 572 void decode_json_obj(obj_version& v, JSONObj *obj);
 573
 574
 575
 576 enum RGWIdentityType
 577 {
 578   TYPE_NONE=0,
 579   TYPE_RGW=1,
 580   TYPE_KEYSTONE=2,
 581   TYPE_LDAP=3,
 582   TYPE_ROLE=4,
 583   TYPE_WEB=5,
 584 };
 585
 586 static std::string RGW_STORAGE_CLASS_STANDARD = "STANDARD";
 587
 588 struct rgw_placement_rule {
 589   std::string name;
 590   std::string storage_class;
 591
 592   rgw_placement_rule() {}
 593   rgw_placement_rule(const std::string& _n, const std::string& _sc) : name(_n), storage_class(_sc) {}
 594   rgw_placement_rule(const rgw_placement_rule& _r, const std::string& _sc) : name(_r.name) {
 595     if (!_sc.empty()) {
 596       storage_class = _sc;
 597     } else {
 598       storage_class = _r.storage_class;
 599     }
 600   }
 601
 602   bool empty() const {
 603     return name.empty() && storage_class.empty();
 604   }
 605
 606   void inherit_from(const rgw_placement_rule& r) {
 607     if (name.empty()) {
 608       name = r.name;
 609     }
 610     if (storage_class.empty()) {
 611       storage_class = r.storage_class;
 612     }
 613   }
 614
 615   void clear() {
 616     name.clear();
 617     storage_class.clear();
 618   }
 619
 620   void init(const std::string& n, const std::string& c) {
 621     name = n;
 622     storage_class = c;
 623   }
 624
 625   static const std::string& get_canonical_storage_class(const std::string& storage_class) {
 626     if (storage_class.empty()) {
 627       return RGW_STORAGE_CLASS_STANDARD;
 628     }
 629     return storage_class;
 630   }
 631
 632   const std::string& get_storage_class() const {
 633     return get_canonical_storage_class(storage_class);
 634   }
 635
 636   int compare(const rgw_placement_rule& r) const {
 637     int c = name.compare(r.name);
 638     if (c != 0) {
 639       return c;
 640     }
 641     return get_storage_class().compare(r.get_storage_class());
 642   }
 643
 644   bool operator==(const rgw_placement_rule& r) const {
 645     return (name == r.name &&
 646             get_storage_class() == r.get_storage_class());
 647   }
 648
 649   bool operator!=(const rgw_placement_rule& r) const {
 650     return !(*this == r);
 651   }
 652
 653   void encode(bufferlist& bl) const {
 654     /* no ENCODE_START/END due to backward compatibility */
 655     std::string s = to_str();
 656     ceph::encode(s, bl);
 657   }
 658
 659   void decode(bufferlist::const_iterator& bl) {
 660     std::string s;
 661     ceph::decode(s, bl);
 662     from_str(s);
 663   }
 664
 665   std::string to_str() const {
 666     if (standard_storage_class()) {
 667       return name;
 668     }
 669     return to_str_explicit();
 670   }
 671
 672   std::string to_str_explicit() const {
 673     return name + "/" + storage_class;
 674   }
 675
 676   void from_str(const std::string& s) {
 677     size_t pos = s.find("/");
 678     if (pos == std::string::npos) {
 679       name = s;
 680       storage_class.clear();
 681       return;
 682     }
 683     name = s.substr(0, pos);
 684     storage_class = s.substr(pos + 1);
 685   }
 686
 687   bool standard_storage_class() const {
 688     return storage_class.empty() || storage_class == RGW_STORAGE_CLASS_STANDARD;
 689   }
 690 };
 691 WRITE_CLASS_ENCODER(rgw_placement_rule)
 692
 693 void encode_json(const char *name, const rgw_placement_rule& val, ceph::Formatter *f);
 694 void decode_json_obj(rgw_placement_rule& v, JSONObj *obj);
 695
 696 inline std::ostream& operator<<(std::ostream& out, const rgw_placement_rule& rule) {
 697   return out << rule.to_str();
 698 }
 699
 700 class RateLimiter;
 701 struct RGWRateLimitInfo {
 702   int64_t max_write_ops;
 703   int64_t max_read_ops;
 704   int64_t max_write_bytes;
 705   int64_t max_read_bytes;
 706   bool enabled = false;
 707   RGWRateLimitInfo()
 708     : max_write_ops(0), max_read_ops(0), max_write_bytes(0), max_read_bytes(0)  {}
 709
 710   void encode(bufferlist& bl) const {
 711     ENCODE_START(1, 1, bl);
 712     encode(max_write_ops, bl);
 713     encode(max_read_ops, bl);
 714     encode(max_write_bytes, bl);
 715     encode(max_read_bytes, bl);
 716     encode(enabled, bl);
 717     ENCODE_FINISH(bl);
 718   }
 719   void decode(bufferlist::const_iterator& bl) {
 720     DECODE_START(1, bl);
 721     decode(max_write_ops,bl);
 722     decode(max_read_ops, bl);
 723     decode(max_write_bytes,bl);
 724     decode(max_read_bytes, bl);
 725     decode(enabled, bl);
 726     DECODE_FINISH(bl);
 727   }
 728
 729   void dump(Formatter *f) const;
 730
 731   void decode_json(JSONObj *obj);
 732
 733 };
 734 WRITE_CLASS_ENCODER(RGWRateLimitInfo)
 735
 736 struct RGWUserInfo
 737 {
 738   rgw_user user_id;
 739   std::string display_name;
 740   std::string user_email;
 741   std::map<std::string, RGWAccessKey> access_keys;
 742   std::map<std::string, RGWAccessKey> swift_keys;
 743   std::map<std::string, RGWSubUser> subusers;
 744   __u8 suspended;
 745   int32_t max_buckets;
 746   uint32_t op_mask;
 747   RGWUserCaps caps;
 748   __u8 admin;
 749   __u8 system;
 750   rgw_placement_rule default_placement;
 751   std::list<std::string> placement_tags;
 752   RGWQuotaInfo bucket_quota;
 753   std::map<int, std::string> temp_url_keys;
 754   RGWQuotaInfo user_quota;
 755   uint32_t type;
 756   std::set<std::string> mfa_ids;
 757   std::string assumed_role_arn;
 758
 759   RGWUserInfo()
 760     : suspended(0),
 761       max_buckets(RGW_DEFAULT_MAX_BUCKETS),
 762       op_mask(RGW_OP_TYPE_ALL),
 763       admin(0),
 764       system(0),
 765       type(TYPE_NONE) {
 766   }
 767
 768   RGWAccessKey* get_key(const std::string& access_key) {
 769     if (access_keys.empty())
 770       return nullptr;
 771
 772     auto k = access_keys.find(access_key);
 773     if (k == access_keys.end())
 774       return nullptr;
 775     else
 776       return &(k->second);
 777   }
 778
 779   void encode(bufferlist& bl) const {
 780      ENCODE_START(22, 9, bl);
 781      encode((uint64_t)0, bl); // old auid
 782      std::string access_key;
 783      std::string secret_key;
 784      if (!access_keys.empty()) {
 785        std::map<std::string, RGWAccessKey>::const_iterator iter = access_keys.begin();
 786        const RGWAccessKey& k = iter->second;
 787        access_key = k.id;
 788        secret_key = k.key;
 789      }
 790      encode(access_key, bl);
 791      encode(secret_key, bl);
 792      encode(display_name, bl);
 793      encode(user_email, bl);
 794      std::string swift_name;
 795      std::string swift_key;
 796      if (!swift_keys.empty()) {
 797        std::map<std::string, RGWAccessKey>::const_iterator iter = swift_keys.begin();
 798        const RGWAccessKey& k = iter->second;
 799        swift_name = k.id;
 800        swift_key = k.key;
 801      }
 802      encode(swift_name, bl);
 803      encode(swift_key, bl);
 804      encode(user_id.id, bl);
 805      encode(access_keys, bl);
 806      encode(subusers, bl);
 807      encode(suspended, bl);
 808      encode(swift_keys, bl);
 809      encode(max_buckets, bl);
 810      encode(caps, bl);
 811      encode(op_mask, bl);
 812      encode(system, bl);
 813      encode(default_placement, bl);
 814      encode(placement_tags, bl);
 815      encode(bucket_quota, bl);
 816      encode(temp_url_keys, bl);
 817      encode(user_quota, bl);
 818      encode(user_id.tenant, bl);
 819      encode(admin, bl);
 820      encode(type, bl);
 821      encode(mfa_ids, bl);
 822      encode(assumed_role_arn, bl);
 823      encode(user_id.ns, bl);
 824      ENCODE_FINISH(bl);
 825   }
 826   void decode(bufferlist::const_iterator& bl) {
 827      DECODE_START_LEGACY_COMPAT_LEN_32(22, 9, 9, bl);
 828      if (struct_v >= 2) {
 829        uint64_t old_auid;
 830        decode(old_auid, bl);
 831      }
 832      std::string access_key;
 833      std::string secret_key;
 834     decode(access_key, bl);
 835     decode(secret_key, bl);
 836     if (struct_v < 6) {
 837       RGWAccessKey k;
 838       k.id = access_key;
 839       k.key = secret_key;
 840       access_keys[access_key] = k;
 841     }
 842     decode(display_name, bl);
 843     decode(user_email, bl);
 844     /* We populate swift_keys map later nowadays, but we have to decode. */
 845     std::string swift_name;
 846     std::string swift_key;
 847     if (struct_v >= 3) decode(swift_name, bl);
 848     if (struct_v >= 4) decode(swift_key, bl);
 849     if (struct_v >= 5)
 850       decode(user_id.id, bl);
 851     else
 852       user_id.id = access_key;
 853     if (struct_v >= 6) {
 854       decode(access_keys, bl);
 855       decode(subusers, bl);
 856     }
 857     suspended = 0;
 858     if (struct_v >= 7) {
 859       decode(suspended, bl);
 860     }
 861     if (struct_v >= 8) {
 862       decode(swift_keys, bl);
 863     }
 864     if (struct_v >= 10) {
 865       decode(max_buckets, bl);
 866     } else {
 867       max_buckets = RGW_DEFAULT_MAX_BUCKETS;
 868     }
 869     if (struct_v >= 11) {
 870       decode(caps, bl);
 871     }
 872     if (struct_v >= 12) {
 873       decode(op_mask, bl);
 874     } else {
 875       op_mask = RGW_OP_TYPE_ALL;
 876     }
 877     if (struct_v >= 13) {
 878       decode(system, bl);
 879       decode(default_placement, bl);
 880       decode(placement_tags, bl); /* tags of allowed placement rules */
 881     }
 882     if (struct_v >= 14) {
 883       decode(bucket_quota, bl);
 884     }
 885     if (struct_v >= 15) {
 886      decode(temp_url_keys, bl);
 887     }
 888     if (struct_v >= 16) {
 889       decode(user_quota, bl);
 890     }
 891     if (struct_v >= 17) {
 892       decode(user_id.tenant, bl);
 893     } else {
 894       user_id.tenant.clear();
 895     }
 896     if (struct_v >= 18) {
 897       decode(admin, bl);
 898     }
 899     if (struct_v >= 19) {
 900       decode(type, bl);
 901     }
 902     if (struct_v >= 20) {
 903       decode(mfa_ids, bl);
 904     }
 905     if (struct_v >= 21) {
 906       decode(assumed_role_arn, bl);
 907     }
 908     if (struct_v >= 22) {
 909       decode(user_id.ns, bl);
 910     } else {
 911       user_id.ns.clear();
 912     }
 913     DECODE_FINISH(bl);
 914   }
 915   void dump(Formatter *f) const;
 916   static void generate_test_instances(std::list<RGWUserInfo*>& o);
 917
 918   void decode_json(JSONObj *obj);
 919 };
 920 WRITE_CLASS_ENCODER(RGWUserInfo)
 921
 922 struct rgw_raw_obj {
 923   rgw_pool pool;
 924   std::string oid;
 925   std::string loc;
 926
 927   rgw_raw_obj() {}
 928   rgw_raw_obj(const rgw_pool& _pool, const std::string& _oid) {
 929     init(_pool, _oid);
 930   }
 931   rgw_raw_obj(const rgw_pool& _pool, const std::string& _oid, const std::string& _loc) : loc(_loc) {
 932     init(_pool, _oid);
 933   }
 934
 935   void init(const rgw_pool& _pool, const std::string& _oid) {
 936     pool = _pool;
 937     oid = _oid;
 938   }
 939
 940   bool empty() const {
 941     return oid.empty();
 942   }
 943
 944   void encode(bufferlist& bl) const {
 945      ENCODE_START(6, 6, bl);
 946     encode(pool, bl);
 947     encode(oid, bl);
 948     encode(loc, bl);
 949     ENCODE_FINISH(bl);
 950   }
 951
 952   void decode_from_rgw_obj(bufferlist::const_iterator& bl);
 953
 954   void decode(bufferlist::const_iterator& bl) {
 955     unsigned ofs = bl.get_off();
 956     DECODE_START(6, bl);
 957     if (struct_v < 6) {
 958       /*
 959        * this object was encoded as rgw_obj, prior to rgw_raw_obj been split out of it,
 960        * let's decode it as rgw_obj and convert it
 961        */
 962       bl.seek(ofs);
 963       decode_from_rgw_obj(bl);
 964       return;
 965     }
 966     decode(pool, bl);
 967     decode(oid, bl);
 968     decode(loc, bl);
 969     DECODE_FINISH(bl);
 970   }
 971
 972   bool operator<(const rgw_raw_obj& o) const {
 973     int r = pool.compare(o.pool);
 974     if (r == 0) {
 975       r = oid.compare(o.oid);
 976       if (r == 0) {
 977         r = loc.compare(o.loc);
 978       }
 979     }
 980     return (r < 0);
 981   }
 982
 983   bool operator==(const rgw_raw_obj& o) const {
 984     return (pool == o.pool && oid == o.oid && loc == o.loc);
 985   }
 986
 987   void dump(Formatter *f) const;
 988   void decode_json(JSONObj *obj);
 989 };
 990 WRITE_CLASS_ENCODER(rgw_raw_obj)
 991
 992 inline std::ostream& operator<<(std::ostream& out, const rgw_raw_obj& o) {
 993   out << o.pool << ":" << o.oid;
 994   return out;
 995 }
 996
 997 struct rgw_bucket_placement {
 998   rgw_placement_rule placement_rule;
 999   rgw_bucket bucket;
1000
1001   void dump(Formatter *f) const;
1002 };
1003
1004 struct RGWObjVersionTracker {
1005   obj_version read_version;
1006   obj_version write_version;
1007
1008   obj_version *version_for_read() {
1009     return &read_version;
1010   }
1011
1012   obj_version *version_for_write() {
1013     if (write_version.ver == 0)
1014       return NULL;
1015
1016     return &write_version;
1017   }
1018
1019   obj_version *version_for_check() {
1020     if (read_version.ver == 0)
1021       return NULL;
1022
1023     return &read_version;
1024   }
1025
1026   void prepare_op_for_read(librados::ObjectReadOperation *op);
1027   void prepare_op_for_write(librados::ObjectWriteOperation *op);
1028
1029   void apply_write();
1030
1031   void clear() {
1032     read_version = obj_version();
1033     write_version = obj_version();
1034   }
1035
1036   void generate_new_write_ver(CephContext *cct);
1037 };
1038
1039 inline std::ostream& operator<<(std::ostream& out, const obj_version &v)
1040 {
1041   out << v.tag << ":" << v.ver;
1042   return out;
1043 }
1044
1045 inline std::ostream& operator<<(std::ostream& out, const RGWObjVersionTracker &ot)
1046 {
1047   out << "{r=" << ot.read_version << ",w=" << ot.write_version << "}";
1048   return out;
1049 }
1050
1051 enum RGWBucketFlags {
1052   BUCKET_SUSPENDED = 0x1,
1053   BUCKET_VERSIONED = 0x2,
1054   BUCKET_VERSIONS_SUSPENDED = 0x4,
1055   BUCKET_DATASYNC_DISABLED = 0X8,
1056   BUCKET_MFA_ENABLED = 0X10,
1057   BUCKET_OBJ_LOCK_ENABLED = 0X20,
1058 };
1059
1060 class RGWSI_Zone;
1061
1062 struct RGWBucketInfo {
1063   rgw_bucket bucket;
1064   rgw_user owner;
1065   uint32_t flags{0};
1066   std::string zonegroup;
1067   ceph::real_time creation_time;
1068   rgw_placement_rule placement_rule;
1069   bool has_instance_obj{false};
1070   RGWObjVersionTracker objv_tracker; /* we don't need to serialize this, for runtime tracking */
1071   RGWQuotaInfo quota;
1072
1073   // layout of bucket index objects
1074   rgw::BucketLayout layout;
1075
1076   // Represents the number of bucket index object shards:
1077   //   - value of 0 indicates there is no sharding (this is by default
1078   //     before this feature is implemented).
1079   //   - value of UINT32_T::MAX indicates this is a blind bucket.
1080
1081   // Represents the shard number for blind bucket.
1082   const static uint32_t NUM_SHARDS_BLIND_BUCKET;
1083
1084   bool requester_pays{false};
1085
1086   bool has_website{false};
1087   RGWBucketWebsiteConf website_conf;
1088
1089   bool swift_versioning{false};
1090   std::string swift_ver_location;
1091
1092   std::map<std::string, uint32_t> mdsearch_config;
1093
1094   // resharding
1095   cls_rgw_reshard_status reshard_status{cls_rgw_reshard_status::NOT_RESHARDING};
1096   std::string new_bucket_instance_id;
1097
1098   RGWObjectLock obj_lock;
1099
1100   std::optional<rgw_sync_policy_info> sync_policy;
1101
1102   void encode(bufferlist& bl) const;
1103   void decode(bufferlist::const_iterator& bl);
1104
1105   void dump(Formatter *f) const;
1106   static void generate_test_instances(std::list<RGWBucketInfo*>& o);
1107
1108   void decode_json(JSONObj *obj);
1109
1110   bool versioned() const { return (flags & BUCKET_VERSIONED) != 0; }
1111   int versioning_status() const { return flags & (BUCKET_VERSIONED | BUCKET_VERSIONS_SUSPENDED | BUCKET_MFA_ENABLED); }
1112   bool versioning_enabled() const { return (versioning_status() & (BUCKET_VERSIONED | BUCKET_VERSIONS_SUSPENDED)) == BUCKET_VERSIONED; }
1113   bool mfa_enabled() const { return (versioning_status() & BUCKET_MFA_ENABLED) != 0; }
1114   bool datasync_flag_enabled() const { return (flags & BUCKET_DATASYNC_DISABLED) == 0; }
1115   bool obj_lock_enabled() const { return (flags & BUCKET_OBJ_LOCK_ENABLED) != 0; }
1116
1117   bool has_swift_versioning() const {
1118     /* A bucket may be versioned through one mechanism only. */
1119     return swift_versioning && !versioned();
1120   }
1121
1122   void set_sync_policy(rgw_sync_policy_info&& policy);
1123
1124   bool empty_sync_policy() const;
1125
1126   RGWBucketInfo();
1127   ~RGWBucketInfo();
1128 };
1129 WRITE_CLASS_ENCODER(RGWBucketInfo)
1130
1131 struct RGWBucketEntryPoint
1132 {
1133   rgw_bucket bucket;
1134   rgw_user owner;
1135   ceph::real_time creation_time;
1136   bool linked;
1137
1138   bool has_bucket_info;
1139   RGWBucketInfo old_bucket_info;
1140
1141   RGWBucketEntryPoint() : linked(false), has_bucket_info(false) {}
1142
1143   void encode(bufferlist& bl) const {
1144     ENCODE_START(10, 8, bl);
1145     encode(bucket, bl);
1146     encode(owner.id, bl);
1147     encode(linked, bl);
1148     uint64_t ctime = (uint64_t)real_clock::to_time_t(creation_time);
1149     encode(ctime, bl);
1150     encode(owner, bl);
1151     encode(creation_time, bl);
1152     ENCODE_FINISH(bl);
1153   }
1154   void decode(bufferlist::const_iterator& bl) {
1155     auto orig_iter = bl;
1156     DECODE_START_LEGACY_COMPAT_LEN_32(10, 4, 4, bl);
1157     if (struct_v < 8) {
1158       /* ouch, old entry, contains the bucket info itself */
1159       old_bucket_info.decode(orig_iter);
1160       has_bucket_info = true;
1161       return;
1162     }
1163     has_bucket_info = false;
1164     decode(bucket, bl);
1165     decode(owner.id, bl);
1166     decode(linked, bl);
1167     uint64_t ctime;
1168     decode(ctime, bl);
1169     if (struct_v < 10) {
1170       creation_time = real_clock::from_time_t((time_t)ctime);
1171     }
1172     if (struct_v >= 9) {
1173       decode(owner, bl);
1174     }
1175     if (struct_v >= 10) {
1176       decode(creation_time, bl);
1177     }
1178     DECODE_FINISH(bl);
1179   }
1180
1181   void dump(Formatter *f) const;
1182   void decode_json(JSONObj *obj);
1183   static void generate_test_instances(std::list<RGWBucketEntryPoint*>& o);
1184 };
1185 WRITE_CLASS_ENCODER(RGWBucketEntryPoint)
1186
1187 struct RGWStorageStats
1188 {
1189   RGWObjCategory category;
1190   uint64_t size;
1191   uint64_t size_rounded;
1192   uint64_t num_objects;
1193   uint64_t size_utilized{0}; //< size after compression, encryption
1194   bool dump_utilized;        // whether dump should include utilized values
1195
1196   RGWStorageStats(bool _dump_utilized=true)
1197     : category(RGWObjCategory::None),
1198       size(0),
1199       size_rounded(0),
1200       num_objects(0),
1201       dump_utilized(_dump_utilized)
1202   {}
1203
1204   void dump(Formatter *f) const;
1205 }; // RGWStorageStats
1206
1207 class RGWEnv;
1208
1209 /* Namespaced forward declarations. */
1210 namespace rgw {
1211   namespace auth {
1212     namespace s3 {
1213       class AWSBrowserUploadAbstractor;
1214       class STSEngine;
1215     }
1216     class Completer;
1217   }
1218   namespace io {
1219     class BasicClient;
1220   }
1221 }
1222
1223 using meta_map_t = boost::container::flat_map <std::string, std::string>;
1224
1225 struct req_info {
1226   const RGWEnv *env;
1227   RGWHTTPArgs args;
1228   meta_map_t x_meta_map;
1229   meta_map_t crypt_attribute_map;
1230
1231   std::string host;
1232   const char *method;
1233   std::string script_uri;
1234   std::string request_uri;
1235   std::string request_uri_aws4;
1236   std::string effective_uri;
1237   std::string request_params;
1238   std::string domain;
1239   std::string storage_class;
1240
1241   req_info(CephContext *cct, const RGWEnv *env);
1242   void rebuild_from(req_info& src);
1243   void init_meta_info(const DoutPrefixProvider *dpp, bool *found_bad_meta);
1244 };
1245
1246 typedef cls_rgw_obj_key rgw_obj_index_key;
1247
1248 struct rgw_obj_key {
1249   std::string name;
1250   std::string instance;
1251   std::string ns;
1252
1253   rgw_obj_key() {}
1254   // cppcheck-suppress noExplicitConstructor
1255   rgw_obj_key(const std::string& n) : name(n) {}
1256   rgw_obj_key(const std::string& n, const std::string& i) : name(n), instance(i) {}
1257   rgw_obj_key(const std::string& n, const std::string& i, const std::string& _ns) : name(n), instance(i), ns(_ns) {}
1258
1259   rgw_obj_key(const rgw_obj_index_key& k) {
1260     parse_index_key(k.name, &name, &ns);
1261     instance = k.instance;
1262   }
1263
1264   static void parse_index_key(const std::string& key, std::string *name, std::string *ns) {
1265     if (key[0] != '_') {
1266       *name = key;
1267       ns->clear();
1268       return;
1269     }
1270     if (key[1] == '_') {
1271       *name = key.substr(1);
1272       ns->clear();
1273       return;
1274     }
1275     ssize_t pos = key.find('_', 1);
1276     if (pos < 0) {
1277       /* shouldn't happen, just use key */
1278       *name = key;
1279       ns->clear();
1280       return;
1281     }
1282
1283     *name = key.substr(pos + 1);
1284     *ns = key.substr(1, pos -1);
1285   }
1286
1287   void set(const std::string& n) {
1288     name = n;
1289     instance.clear();
1290     ns.clear();
1291   }
1292
1293   void set(const std::string& n, const std::string& i) {
1294     name = n;
1295     instance = i;
1296     ns.clear();
1297   }
1298
1299   void set(const std::string& n, const std::string& i, const std::string& _ns) {
1300     name = n;
1301     instance = i;
1302     ns = _ns;
1303   }
1304
1305   bool set(const rgw_obj_index_key& index_key) {
1306     if (!parse_raw_oid(index_key.name, this)) {
1307       return false;
1308     }
1309     instance = index_key.instance;
1310     return true;
1311   }
1312
1313   void set_instance(const std::string& i) {
1314     instance = i;
1315   }
1316
1317   const std::string& get_instance() const {
1318     return instance;
1319   }
1320
1321   void set_ns(const std::string& _ns) {
1322     ns = _ns;
1323   }
1324
1325   const std::string& get_ns() const {
1326     return ns;
1327   }
1328
1329   std::string get_index_key_name() const {
1330     if (ns.empty()) {
1331       if (name.size() < 1 || name[0] != '_') {
1332         return name;
1333       }
1334       return std::string("_") + name;
1335     };
1336
1337     char buf[ns.size() + 16];
1338     snprintf(buf, sizeof(buf), "_%s_", ns.c_str());
1339     return std::string(buf) + name;
1340   };
1341
1342   void get_index_key(rgw_obj_index_key *key) const {
1343     key->name = get_index_key_name();
1344     key->instance = instance;
1345   }
1346
1347   std::string get_loc() const {
1348     /*
1349      * For backward compatibility. Older versions used to have object locator on all objects,
1350      * however, the name was the effective object locator. This had the same effect as not
1351      * having object locator at all for most objects but the ones that started with underscore as
1352      * these were escaped.
1353      */
1354     if (name[0] == '_' && ns.empty()) {
1355       return name;
1356     }
1357
1358     return {};
1359   }
1360
1361   bool empty() const {
1362     return name.empty();
1363   }
1364
1365   bool have_null_instance() const {
1366     return instance == "null";
1367   }
1368
1369   bool have_instance() const {
1370     return !instance.empty();
1371   }
1372
1373   bool need_to_encode_instance() const {
1374     return have_instance() && !have_null_instance();
1375   }
1376
1377   std::string get_oid() const {
1378     if (ns.empty() && !need_to_encode_instance()) {
1379       if (name.size() < 1 || name[0] != '_') {
1380         return name;
1381       }
1382       return std::string("_") + name;
1383     }
1384
1385     std::string oid = "_";
1386     oid.append(ns);
1387     if (need_to_encode_instance()) {
1388       oid.append(std::string(":") + instance);
1389     }
1390     oid.append("_");
1391     oid.append(name);
1392     return oid;
1393   }
1394
1395   bool operator==(const rgw_obj_key& k) const {
1396     return (name.compare(k.name) == 0) &&
1397            (instance.compare(k.instance) == 0);
1398   }
1399
1400   bool operator<(const rgw_obj_key& k) const {
1401     int r = name.compare(k.name);
1402     if (r == 0) {
1403       r = instance.compare(k.instance);
1404     }
1405     return (r < 0);
1406   }
1407
1408   bool operator<=(const rgw_obj_key& k) const {
1409     return !(k < *this);
1410   }
1411
1412   static void parse_ns_field(std::string& ns, std::string& instance) {
1413     int pos = ns.find(':');
1414     if (pos >= 0) {
1415       instance = ns.substr(pos + 1);
1416       ns = ns.substr(0, pos);
1417     } else {
1418       instance.clear();
1419     }
1420   }
1421
1422   // takes an oid and parses out the namespace (ns), name, and
1423   // instance
1424   static bool parse_raw_oid(const std::string& oid, rgw_obj_key *key) {
1425     key->instance.clear();
1426     key->ns.clear();
1427     if (oid[0] != '_') {
1428       key->name = oid;
1429       return true;
1430     }
1431
1432     if (oid.size() >= 2 && oid[1] == '_') {
1433       key->name = oid.substr(1);
1434       return true;
1435     }
1436
1437     if (oid.size() < 3) // for namespace, min size would be 3: _x_
1438       return false;
1439
1440     size_t pos = oid.find('_', 2); // oid must match ^_[^_].+$
1441     if (pos == std::string::npos)
1442       return false;
1443
1444     key->ns = oid.substr(1, pos - 1);
1445     parse_ns_field(key->ns, key->instance);
1446
1447     key->name = oid.substr(pos + 1);
1448     return true;
1449   }
1450
1451   /**
1452    * Translate a namespace-mangled object name to the user-facing name
1453    * existing in the given namespace.
1454    *
1455    * If the object is part of the given namespace, it returns true
1456    * and cuts down the name to the unmangled version. If it is not
1457    * part of the given namespace, it returns false.
1458    */
1459   static bool oid_to_key_in_ns(const std::string& oid, rgw_obj_key *key, const std::string& ns) {
1460     bool ret = parse_raw_oid(oid, key);
1461     if (!ret) {
1462       return ret;
1463     }
1464
1465     return (ns == key->ns);
1466   }
1467
1468   /**
1469    * Given a mangled object name and an empty namespace std::string, this
1470    * function extracts the namespace into the std::string and sets the object
1471    * name to be the unmangled version.
1472    *
1473    * It returns true after successfully doing so, or
1474    * false if it fails.
1475    */
1476   static bool strip_namespace_from_name(std::string& name, std::string& ns, std::string& instance) {
1477     ns.clear();
1478     instance.clear();
1479     if (name[0] != '_') {
1480       return true;
1481     }
1482
1483     size_t pos = name.find('_', 1);
1484     if (pos == std::string::npos) {
1485       return false;
1486     }
1487
1488     if (name[1] == '_') {
1489       name = name.substr(1);
1490       return true;
1491     }
1492
1493     size_t period_pos = name.find('.');
1494     if (period_pos < pos) {
1495       return false;
1496     }
1497
1498     ns = name.substr(1, pos-1);
1499     name = name.substr(pos+1, std::string::npos);
1500
1501     parse_ns_field(ns, instance);
1502     return true;
1503   }
1504
1505   void encode(bufferlist& bl) const {
1506     ENCODE_START(2, 1, bl);
1507     encode(name, bl);
1508     encode(instance, bl);
1509     encode(ns, bl);
1510     ENCODE_FINISH(bl);
1511   }
1512   void decode(bufferlist::const_iterator& bl) {
1513     DECODE_START(2, bl);
1514     decode(name, bl);
1515     decode(instance, bl);
1516     if (struct_v >= 2) {
1517       decode(ns, bl);
1518     }
1519     DECODE_FINISH(bl);
1520   }
1521   void dump(Formatter *f) const;
1522   void decode_json(JSONObj *obj);
1523
1524   std::string to_str() const {
1525     if (instance.empty()) {
1526       return name;
1527     }
1528     char buf[name.size() + instance.size() + 16];
1529     snprintf(buf, sizeof(buf), "%s[%s]", name.c_str(), instance.c_str());
1530     return buf;
1531   }
1532 };
1533 WRITE_CLASS_ENCODER(rgw_obj_key)
1534
1535 inline std::ostream& operator<<(std::ostream& out, const rgw_obj_key &o) {
1536   return out << o.to_str();
1537 }
1538
1539 struct req_init_state {
1540   /* Keeps [[tenant]:]bucket until we parse the token. */
1541   std::string url_bucket;
1542   std::string src_bucket;
1543 };
1544
1545 #include "rgw_auth.h"
1546
1547 class RGWObjectCtx;
1548 class RGWSysObjectCtx;
1549
1550 /** Store all the state necessary to complete and respond to an HTTP request*/
1551 struct req_state : DoutPrefixProvider {
1552   CephContext *cct;
1553   rgw::io::BasicClient *cio{nullptr};
1554   http_op op{OP_UNKNOWN};
1555   RGWOpType op_type{};
1556   std::shared_ptr<RateLimiter> ratelimit_data;
1557   RGWRateLimitInfo user_ratelimit;
1558   RGWRateLimitInfo bucket_ratelimit;
1559   std::string ratelimit_bucket_marker;
1560   std::string ratelimit_user_name;
1561   bool content_started{false};
1562   int format{0};
1563   ceph::Formatter *formatter{nullptr};
1564   std::string decoded_uri;
1565   std::string relative_uri;
1566   const char *length{nullptr};
1567   int64_t content_length{0};
1568   std::map<std::string, std::string> generic_attrs;
1569   rgw_err err;
1570   bool expect_cont{false};
1571   uint64_t obj_size{0};
1572   bool enable_ops_log;
1573   bool enable_usage_log;
1574   uint8_t defer_to_bucket_acls;
1575   uint32_t perm_mask{0};
1576
1577   /* Set once when url_bucket is parsed and not violated thereafter. */
1578   std::string account_name;
1579
1580   std::string bucket_tenant;
1581   std::string bucket_name;
1582
1583   /* bucket is only created in rgw_build_bucket_policies() and should never be
1584    * overwritten */
1585   std::unique_ptr<rgw::sal::Bucket> bucket;
1586   std::unique_ptr<rgw::sal::Object> object;
1587   std::string src_tenant_name;
1588   std::string src_bucket_name;
1589   std::unique_ptr<rgw::sal::Object> src_object;
1590   ACLOwner bucket_owner;
1591   ACLOwner owner;
1592
1593   std::string zonegroup_name;
1594   std::string zonegroup_endpoint;
1595   std::string bucket_instance_id;
1596   int bucket_instance_shard_id{-1};
1597   std::string redirect_zone_endpoint;
1598
1599   std::string redirect;
1600
1601   real_time bucket_mtime;
1602   std::map<std::string, ceph::bufferlist> bucket_attrs;
1603   bool bucket_exists{false};
1604   rgw_placement_rule dest_placement;
1605
1606   bool has_bad_meta{false};
1607
1608   std::unique_ptr<rgw::sal::User> user;
1609
1610   struct {
1611     /* TODO(rzarzynski): switch out to the static_ptr for both members. */
1612
1613     /* Object having the knowledge about an authenticated identity and allowing
1614      * to apply it during the authorization phase (verify_permission() methods
1615      * of a given RGWOp). Thus, it bounds authentication and authorization steps
1616      * through a well-defined interface. For more details, see rgw_auth.h. */
1617     std::unique_ptr<rgw::auth::Identity> identity;
1618
1619     std::shared_ptr<rgw::auth::Completer> completer;
1620
1621     /* A container for credentials of the S3's browser upload. It's necessary
1622      * because: 1) the ::authenticate() method of auth engines and strategies
1623      * take req_state only; 2) auth strategies live much longer than RGWOps -
1624      * there is no way to pass additional data dependencies through ctors. */
1625     class {
1626       /* Writer. */
1627       friend class RGWPostObj_ObjStore_S3;
1628       /* Reader. */
1629       friend class rgw::auth::s3::AWSBrowserUploadAbstractor;
1630       friend class rgw::auth::s3::STSEngine;
1631
1632       std::string access_key;
1633       std::string signature;
1634       std::string x_amz_algorithm;
1635       std::string x_amz_credential;
1636       std::string x_amz_date;
1637       std::string x_amz_security_token;
1638       ceph::bufferlist encoded_policy;
1639     } s3_postobj_creds;
1640   } auth;
1641
1642   std::unique_ptr<RGWAccessControlPolicy> user_acl;
1643   std::unique_ptr<RGWAccessControlPolicy> bucket_acl;
1644   std::unique_ptr<RGWAccessControlPolicy> object_acl;
1645
1646   rgw::IAM::Environment env;
1647   boost::optional<rgw::IAM::Policy> iam_policy;
1648   boost::optional<PublicAccessBlockConfiguration> bucket_access_conf;
1649   std::vector<rgw::IAM::Policy> iam_user_policies;
1650
1651   /* Is the request made by an user marked as a system one?
1652    * Being system user means we also have the admin status. */
1653   bool system_request{false};
1654
1655   std::string canned_acl;
1656   bool has_acl_header{false};
1657   bool local_source{false}; /* source is local */
1658
1659   int prot_flags{0};
1660
1661   /* Content-Disposition override for TempURL of Swift API. */
1662   struct {
1663     std::string override;
1664     std::string fallback;
1665   } content_disp;
1666
1667   std::string host_id;
1668
1669   req_info info;
1670   req_init_state init_state;
1671
1672   using Clock = ceph::coarse_real_clock;
1673   Clock::time_point time;
1674
1675   Clock::duration time_elapsed() const { return Clock::now() - time; }
1676
1677   RGWObjectCtx *obj_ctx{nullptr};
1678   std::string dialect;
1679   std::string req_id;
1680   std::string trans_id;
1681   uint64_t id;
1682
1683   RGWObjTags tagset;
1684
1685   bool mfa_verified{false};
1686
1687   /// optional coroutine context
1688   optional_yield yield{null_yield};
1689
1690   //token claims from STS token for ops log (can be used for Keystone token also)
1691   std::vector<std::string> token_claims;
1692
1693   std::vector<rgw::IAM::Policy> session_policies;
1694
1695   jspan trace;
1696
1697   //Principal tags that come in as part of AssumeRoleWithWebIdentity
1698   std::vector<std::pair<std::string, std::string>> principal_tags;
1699
1700   req_state(CephContext* _cct, RGWEnv* e, uint64_t id);
1701   ~req_state();
1702
1703
1704   void set_user(std::unique_ptr<rgw::sal::User>& u) { user.swap(u); }
1705   bool is_err() const { return err.is_err(); }
1706
1707   // implements DoutPrefixProvider
1708   std::ostream& gen_prefix(std::ostream& out) const override;
1709   CephContext* get_cct() const override { return cct; }
1710   unsigned get_subsys() const override { return ceph_subsys_rgw; }
1711 };
1712
1713 void set_req_state_err(struct req_state*, int);
1714 void set_req_state_err(struct req_state*, int, const std::string&);
1715 void set_req_state_err(struct rgw_err&, int, const int);
1716 void dump(struct req_state*);
1717
1718 /** Store basic data on bucket */
1719 struct RGWBucketEnt {
1720   rgw_bucket bucket;
1721   size_t size;
1722   size_t size_rounded;
1723   ceph::real_time creation_time;
1724   uint64_t count;
1725
1726   /* The placement_rule is necessary to calculate per-storage-policy statics
1727    * of the Swift API. Although the info available in RGWBucketInfo, we need
1728    * to duplicate it here to not affect the performance of buckets listing. */
1729   rgw_placement_rule placement_rule;
1730
1731   RGWBucketEnt()
1732     : size(0),
1733       size_rounded(0),
1734       count(0) {
1735   }
1736   RGWBucketEnt(const RGWBucketEnt&) = default;
1737   RGWBucketEnt(RGWBucketEnt&&) = default;
1738   explicit RGWBucketEnt(const rgw_user& u, cls_user_bucket_entry&& e)
1739     : bucket(u, std::move(e.bucket)),
1740       size(e.size),
1741       size_rounded(e.size_rounded),
1742       creation_time(e.creation_time),
1743       count(e.count) {
1744   }
1745
1746   RGWBucketEnt& operator=(const RGWBucketEnt&) = default;
1747
1748   void convert(cls_user_bucket_entry *b) const {
1749     bucket.convert(&b->bucket);
1750     b->size = size;
1751     b->size_rounded = size_rounded;
1752     b->creation_time = creation_time;
1753     b->count = count;
1754   }
1755
1756   void encode(bufferlist& bl) const {
1757     ENCODE_START(7, 5, bl);
1758     uint64_t s = size;
1759     __u32 mt = ceph::real_clock::to_time_t(creation_time);
1760     std::string empty_str;  // originally had the bucket name here, but we encode bucket later
1761     encode(empty_str, bl);
1762     encode(s, bl);
1763     encode(mt, bl);
1764     encode(count, bl);
1765     encode(bucket, bl);
1766     s = size_rounded;
1767     encode(s, bl);
1768     encode(creation_time, bl);
1769     encode(placement_rule, bl);
1770     ENCODE_FINISH(bl);
1771   }
1772   void decode(bufferlist::const_iterator& bl) {
1773     DECODE_START_LEGACY_COMPAT_LEN(7, 5, 5, bl);
1774     __u32 mt;
1775     uint64_t s;
1776     std::string empty_str;  // backward compatibility
1777     decode(empty_str, bl);
1778     decode(s, bl);
1779     decode(mt, bl);
1780     size = s;
1781     if (struct_v < 6) {
1782       creation_time = ceph::real_clock::from_time_t(mt);
1783     }
1784     if (struct_v >= 2)
1785       decode(count, bl);
1786     if (struct_v >= 3)
1787       decode(bucket, bl);
1788     if (struct_v >= 4)
1789       decode(s, bl);
1790     size_rounded = s;
1791     if (struct_v >= 6)
1792       decode(creation_time, bl);
1793     if (struct_v >= 7)
1794       decode(placement_rule, bl);
1795     DECODE_FINISH(bl);
1796   }
1797   void dump(Formatter *f) const;
1798   static void generate_test_instances(std::list<RGWBucketEnt*>& o);
1799 };
1800 WRITE_CLASS_ENCODER(RGWBucketEnt)
1801
1802 struct rgw_obj {
1803   rgw_bucket bucket;
1804   rgw_obj_key key;
1805
1806   bool in_extra_data{false}; /* in-memory only member, does not serialize */
1807
1808   // Represents the hash index source for this object once it is set (non-empty)
1809   std::string index_hash_source;
1810
1811   rgw_obj() {}
1812   rgw_obj(const rgw_bucket& b, const std::string& name) : bucket(b), key(name) {}
1813   rgw_obj(const rgw_bucket& b, const rgw_obj_key& k) : bucket(b), key(k) {}
1814   rgw_obj(const rgw_bucket& b, const rgw_obj_index_key& k) : bucket(b), key(k) {}
1815
1816   void init(const rgw_bucket& b, const std::string& name) {
1817     bucket = b;
1818     key.set(name);
1819   }
1820   void init(const rgw_bucket& b, const std::string& name, const std::string& i, const std::string& n) {
1821     bucket = b;
1822     key.set(name, i, n);
1823   }
1824   void init_ns(const rgw_bucket& b, const std::string& name, const std::string& n) {
1825     bucket = b;
1826     key.name = name;
1827     key.instance.clear();
1828     key.ns = n;
1829   }
1830
1831   bool empty() const {
1832     return key.empty();
1833   }
1834
1835   void set_key(const rgw_obj_key& k) {
1836     key = k;
1837   }
1838
1839   std::string get_oid() const {
1840     return key.get_oid();
1841   }
1842
1843   const std::string& get_hash_object() const {
1844     return index_hash_source.empty() ? key.name : index_hash_source;
1845   }
1846
1847   void set_in_extra_data(bool val) {
1848     in_extra_data = val;
1849   }
1850
1851   bool is_in_extra_data() const {
1852     return in_extra_data;
1853   }
1854
1855   void encode(bufferlist& bl) const {
1856     ENCODE_START(6, 6, bl);
1857     encode(bucket, bl);
1858     encode(key.ns, bl);
1859     encode(key.name, bl);
1860     encode(key.instance, bl);
1861 //    encode(placement_id, bl);
1862     ENCODE_FINISH(bl);
1863   }
1864   void decode(bufferlist::const_iterator& bl) {
1865     DECODE_START_LEGACY_COMPAT_LEN(6, 3, 3, bl);
1866     if (struct_v < 6) {
1867       std::string s;
1868       decode(bucket.name, bl); /* bucket.name */
1869       decode(s, bl); /* loc */
1870       decode(key.ns, bl);
1871       decode(key.name, bl);
1872       if (struct_v >= 2)
1873         decode(bucket, bl);
1874       if (struct_v >= 4)
1875         decode(key.instance, bl);
1876       if (key.ns.empty() && key.instance.empty()) {
1877         if (key.name[0] == '_') {
1878           key.name = key.name.substr(1);
1879         }
1880       } else {
1881         if (struct_v >= 5) {
1882           decode(key.name, bl);
1883         } else {
1884           ssize_t pos = key.name.find('_', 1);
1885           if (pos < 0) {
1886             throw buffer::malformed_input();
1887           }
1888           key.name = key.name.substr(pos + 1);
1889         }
1890       }
1891     } else {
1892       decode(bucket, bl);
1893       decode(key.ns, bl);
1894       decode(key.name, bl);
1895       decode(key.instance, bl);
1896 //      decode(placement_id, bl);
1897     }
1898     DECODE_FINISH(bl);
1899   }
1900   void dump(Formatter *f) const;
1901   static void generate_test_instances(std::list<rgw_obj*>& o);
1902
1903   bool operator==(const rgw_obj& o) const {
1904     return (key == o.key) &&
1905            (bucket == o.bucket);
1906   }
1907   bool operator<(const rgw_obj& o) const {
1908     int r = key.name.compare(o.key.name);
1909     if (r == 0) {
1910       r = bucket.bucket_id.compare(o.bucket.bucket_id); /* not comparing bucket.name, if bucket_id is equal so will be bucket.name */
1911       if (r == 0) {
1912         r = key.ns.compare(o.key.ns);
1913         if (r == 0) {
1914           r = key.instance.compare(o.key.instance);
1915         }
1916       }
1917     }
1918
1919     return (r < 0);
1920   }
1921
1922   const rgw_pool& get_explicit_data_pool() {
1923     if (!in_extra_data || bucket.explicit_placement.data_extra_pool.empty()) {
1924       return bucket.explicit_placement.data_pool;
1925     }
1926     return bucket.explicit_placement.data_extra_pool;
1927   }
1928 };
1929 WRITE_CLASS_ENCODER(rgw_obj)
1930
1931 struct rgw_cache_entry_info {
1932   std::string cache_locator;
1933   uint64_t gen;
1934
1935   rgw_cache_entry_info() : gen(0) {}
1936 };
1937
1938 inline std::ostream& operator<<(std::ostream& out, const rgw_obj &o) {
1939   return out << o.bucket.name << ":" << o.get_oid();
1940 }
1941
1942 struct multipart_upload_info
1943 {
1944   rgw_placement_rule dest_placement;
1945
1946   void encode(bufferlist& bl) const {
1947     ENCODE_START(1, 1, bl);
1948     encode(dest_placement, bl);
1949     ENCODE_FINISH(bl);
1950   }
1951
1952   void decode(bufferlist::const_iterator& bl) {
1953     DECODE_START(1, bl);
1954     decode(dest_placement, bl);
1955     DECODE_FINISH(bl);
1956   }
1957 };
1958 WRITE_CLASS_ENCODER(multipart_upload_info)
1959
1960 static inline void buf_to_hex(const unsigned char* const buf,
1961                               const size_t len,
1962                               char* const str)
1963 {
1964   str[0] = '\0';
1965   for (size_t i = 0; i < len; i++) {
1966     ::sprintf(&str[i*2], "%02x", static_cast<int>(buf[i]));
1967   }
1968 }
1969
1970 template<size_t N> static inline std::array<char, N * 2 + 1>
1971 buf_to_hex(const std::array<unsigned char, N>& buf)
1972 {
1973   static_assert(N > 0, "The input array must be at least one element long");
1974
1975   std::array<char, N * 2 + 1> hex_dest;
1976   buf_to_hex(buf.data(), N, hex_dest.data());
1977   return hex_dest;
1978 }
1979
1980 static inline int hexdigit(char c)
1981 {
1982   if (c >= '0' && c <= '9')
1983     return (c - '0');
1984   c = toupper(c);
1985   if (c >= 'A' && c <= 'F')
1986     return c - 'A' + 0xa;
1987   return -EINVAL;
1988 }
1989
1990 static inline int hex_to_buf(const char *hex, char *buf, int len)
1991 {
1992   int i = 0;
1993   const char *p = hex;
1994   while (*p) {
1995     if (i >= len)
1996       return -EINVAL;
1997     buf[i] = 0;
1998     int d = hexdigit(*p);
1999     if (d < 0)
2000       return d;
2001     buf[i] = d << 4;
2002     p++;
2003     if (!*p)
2004       return -EINVAL;
2005     d = hexdigit(*p);
2006     if (d < 0)
2007       return d;
2008     buf[i] += d;
2009     i++;
2010     p++;
2011   }
2012   return i;
2013 }
2014
2015 static inline int rgw_str_to_bool(const char *s, int def_val)
2016 {
2017   if (!s)
2018     return def_val;
2019
2020   return (strcasecmp(s, "true") == 0 ||
2021           strcasecmp(s, "on") == 0 ||
2022           strcasecmp(s, "yes") == 0 ||
2023           strcasecmp(s, "1") == 0);
2024 }
2025
2026 static inline void append_rand_alpha(CephContext *cct, const std::string& src, std::string& dest, int len)
2027 {
2028   dest = src;
2029   char buf[len + 1];
2030   gen_rand_alphanumeric(cct, buf, len);
2031   dest.append("_");
2032   dest.append(buf);
2033 }
2034
2035 static inline uint64_t rgw_rounded_kb(uint64_t bytes)
2036 {
2037   return (bytes + 1023) / 1024;
2038 }
2039
2040 static inline uint64_t rgw_rounded_objsize(uint64_t bytes)
2041 {
2042   return ((bytes + 4095) & ~4095);
2043 }
2044
2045 static inline uint64_t rgw_rounded_objsize_kb(uint64_t bytes)
2046 {
2047   return ((bytes + 4095) & ~4095) / 1024;
2048 }
2049
2050 /* implement combining step, S3 header canonicalization;  k is a
2051  * valid header and in lc form */
2052 void rgw_add_amz_meta_header(
2053   meta_map_t& x_meta_map,
2054   const std::string& k,
2055   const std::string& v);
2056
2057 enum rgw_set_action_if_set {
2058   DISCARD=0, OVERWRITE, APPEND
2059 };
2060
2061 bool rgw_set_amz_meta_header(
2062   meta_map_t& x_meta_map,
2063   const std::string& k,
2064   const std::string& v, rgw_set_action_if_set f);
2065
2066 extern std::string rgw_string_unquote(const std::string& s);
2067 extern void parse_csv_string(const std::string& ival, std::vector<std::string>& ovals);
2068 extern int parse_key_value(std::string& in_str, std::string& key, std::string& val);
2069 extern int parse_key_value(std::string& in_str, const char *delim, std::string& key, std::string& val);
2070
2071 extern boost::optional<std::pair<std::string_view,std::string_view>>
2072 parse_key_value(const std::string_view& in_str,
2073                 const std::string_view& delim);
2074 extern boost::optional<std::pair<std::string_view,std::string_view>>
2075 parse_key_value(const std::string_view& in_str);
2076
2077
2078 /** time parsing */
2079 extern int parse_time(const char *time_str, real_time *time);
2080 extern bool parse_rfc2616(const char *s, struct tm *t);
2081 extern bool parse_iso8601(const char *s, struct tm *t, uint32_t *pns = NULL, bool extended_format = true);
2082 extern std::string rgw_trim_whitespace(const std::string& src);
2083 extern std::string_view rgw_trim_whitespace(const std::string_view& src);
2084 extern std::string rgw_trim_quotes(const std::string& val);
2085
2086 extern void rgw_to_iso8601(const real_time& t, char *dest, int buf_size);
2087 extern void rgw_to_iso8601(const real_time& t, std::string *dest);
2088 extern std::string rgw_to_asctime(const utime_t& t);
2089
2090 struct perm_state_base {
2091   CephContext *cct;
2092   const rgw::IAM::Environment& env;
2093   rgw::auth::Identity *identity;
2094   const RGWBucketInfo bucket_info;
2095   int perm_mask;
2096   bool defer_to_bucket_acls;
2097   boost::optional<PublicAccessBlockConfiguration> bucket_access_conf;
2098
2099   perm_state_base(CephContext *_cct,
2100                   const rgw::IAM::Environment& _env,
2101                   rgw::auth::Identity *_identity,
2102                   const RGWBucketInfo& _bucket_info,
2103                   int _perm_mask,
2104                   bool _defer_to_bucket_acls,
2105                   boost::optional<PublicAccessBlockConfiguration> _bucket_acess_conf = boost::none) :
2106                                                 cct(_cct),
2107                                                 env(_env),
2108                                                 identity(_identity),
2109                                                 bucket_info(_bucket_info),
2110                                                 perm_mask(_perm_mask),
2111                                                 defer_to_bucket_acls(_defer_to_bucket_acls),
2112                                                 bucket_access_conf(_bucket_acess_conf)
2113   {}
2114
2115   virtual ~perm_state_base() {}
2116
2117   virtual const char *get_referer() const = 0;
2118   virtual std::optional<bool> get_request_payer() const = 0; /*
2119                                                               * empty state means that request_payer param was not passed in
2120                                                               */
2121
2122 };
2123
2124 struct perm_state : public perm_state_base {
2125   const char *referer;
2126   bool request_payer;
2127
2128   perm_state(CephContext *_cct,
2129              const rgw::IAM::Environment& _env,
2130              rgw::auth::Identity *_identity,
2131              const RGWBucketInfo& _bucket_info,
2132              int _perm_mask,
2133              bool _defer_to_bucket_acls,
2134              const char *_referer,
2135              bool _request_payer) : perm_state_base(_cct,
2136                                                     _env,
2137                                                     _identity,
2138                                                     _bucket_info,
2139                                                     _perm_mask,
2140                                                     _defer_to_bucket_acls),
2141                                     referer(_referer),
2142                                     request_payer(_request_payer) {}
2143
2144   const char *get_referer() const override {
2145     return referer;
2146   }
2147
2148   std::optional<bool> get_request_payer() const override {
2149     return request_payer;
2150   }
2151 };
2152
2153 /** Check if the req_state's user has the necessary permissions
2154  * to do the requested action  */
2155 bool verify_bucket_permission_no_policy(
2156   const DoutPrefixProvider* dpp,
2157   struct perm_state_base * const s,
2158   RGWAccessControlPolicy * const user_acl,
2159   RGWAccessControlPolicy * const bucket_acl,
2160   const int perm);
2161
2162 bool verify_user_permission_no_policy(const DoutPrefixProvider* dpp,
2163                                       struct perm_state_base * const s,
2164                                       RGWAccessControlPolicy * const user_acl,
2165                                       const int perm);
2166
2167 bool verify_object_permission_no_policy(const DoutPrefixProvider* dpp,
2168                                         struct perm_state_base * const s,
2169                                         RGWAccessControlPolicy * const user_acl,
2170                                         RGWAccessControlPolicy * const bucket_acl,
2171                                         RGWAccessControlPolicy * const object_acl,
2172                                         const int perm);
2173
2174 /** Check if the req_state's user has the necessary permissions
2175  * to do the requested action */
2176 rgw::IAM::Effect eval_identity_or_session_policies(const std::vector<rgw::IAM::Policy>& user_policies,
2177                           const rgw::IAM::Environment& env,
2178                           const uint64_t op,
2179                           const rgw::ARN& arn);
2180 bool verify_user_permission(const DoutPrefixProvider* dpp,
2181                             struct req_state * const s,
2182                             RGWAccessControlPolicy * const user_acl,
2183                             const std::vector<rgw::IAM::Policy>& user_policies,
2184                             const std::vector<rgw::IAM::Policy>& session_policies,
2185                             const rgw::ARN& res,
2186                             const uint64_t op);
2187 bool verify_user_permission_no_policy(const DoutPrefixProvider* dpp,
2188                                       struct req_state * const s,
2189                                       RGWAccessControlPolicy * const user_acl,
2190                                       const int perm);
2191 bool verify_user_permission(const DoutPrefixProvider* dpp,
2192                             struct req_state * const s,
2193                             const rgw::ARN& res,
2194                             const uint64_t op);
2195 bool verify_user_permission_no_policy(const DoutPrefixProvider* dpp,
2196                                       struct req_state * const s,
2197                                       int perm);
2198 bool verify_bucket_permission(
2199   const DoutPrefixProvider* dpp,
2200   struct req_state * const s,
2201   const rgw_bucket& bucket,
2202   RGWAccessControlPolicy * const user_acl,
2203   RGWAccessControlPolicy * const bucket_acl,
2204   const boost::optional<rgw::IAM::Policy>& bucket_policy,
2205   const std::vector<rgw::IAM::Policy>& identity_policies,
2206   const std::vector<rgw::IAM::Policy>& session_policies,
2207   const uint64_t op);
2208 bool verify_bucket_permission(const DoutPrefixProvider* dpp, struct req_state * const s, const uint64_t op);
2209 bool verify_bucket_permission_no_policy(
2210   const DoutPrefixProvider* dpp,
2211   struct req_state * const s,
2212   RGWAccessControlPolicy * const user_acl,
2213   RGWAccessControlPolicy * const bucket_acl,
2214   const int perm);
2215 bool verify_bucket_permission_no_policy(const DoutPrefixProvider* dpp,
2216                                         struct req_state * const s,
2217                                         const int perm);
2218 int verify_bucket_owner_or_policy(struct req_state* const s,
2219                                   const uint64_t op);
2220 extern bool verify_object_permission(
2221   const DoutPrefixProvider* dpp,
2222   struct req_state * const s,
2223   const rgw_obj& obj,
2224   RGWAccessControlPolicy * const user_acl,
2225   RGWAccessControlPolicy * const bucket_acl,
2226   RGWAccessControlPolicy * const object_acl,
2227   const boost::optional<rgw::IAM::Policy>& bucket_policy,
2228   const std::vector<rgw::IAM::Policy>& identity_policies,
2229   const std::vector<rgw::IAM::Policy>& session_policies,
2230   const uint64_t op);
2231 extern bool verify_object_permission(const DoutPrefixProvider* dpp, struct req_state *s, uint64_t op);
2232 extern bool verify_object_permission_no_policy(
2233   const DoutPrefixProvider* dpp,
2234   struct req_state * const s,
2235   RGWAccessControlPolicy * const user_acl,
2236   RGWAccessControlPolicy * const bucket_acl,
2237   RGWAccessControlPolicy * const object_acl,
2238   int perm);
2239 extern bool verify_object_permission_no_policy(const DoutPrefixProvider* dpp, struct req_state *s,
2240                                                int perm);
2241 extern int verify_object_lock(
2242   const DoutPrefixProvider* dpp,
2243   const rgw::sal::Attrs& attrs,
2244   const bool bypass_perm,
2245   const bool bypass_governance_mode);
2246
2247 /** Convert an input URL into a sane object name
2248  * by converting %-escaped std::strings into characters, etc*/
2249 extern void rgw_uri_escape_char(char c, std::string& dst);
2250 extern std::string url_decode(const std::string_view& src_str,
2251                               bool in_query = false);
2252 extern void url_encode(const std::string& src, std::string& dst,
2253                        bool encode_slash = true);
2254 extern std::string url_encode(const std::string& src, bool encode_slash = true);
2255 extern std::string url_remove_prefix(const std::string& url); // Removes hhtp, https and www from url
2256 /* destination should be CEPH_CRYPTO_HMACSHA1_DIGESTSIZE bytes long */
2257 extern void calc_hmac_sha1(const char *key, int key_len,
2258                           const char *msg, int msg_len, char *dest);
2259
2260 static inline sha1_digest_t
2261 calc_hmac_sha1(const std::string_view& key, const std::string_view& msg) {
2262   sha1_digest_t dest;
2263   calc_hmac_sha1(key.data(), key.size(), msg.data(), msg.size(),
2264                  reinterpret_cast<char*>(dest.v));
2265   return dest;
2266 }
2267
2268 /* destination should be CEPH_CRYPTO_HMACSHA256_DIGESTSIZE bytes long */
2269 extern void calc_hmac_sha256(const char *key, int key_len,
2270                              const char *msg, int msg_len,
2271                              char *dest);
2272
2273 static inline sha256_digest_t
2274 calc_hmac_sha256(const char *key, const int key_len,
2275                  const char *msg, const int msg_len) {
2276   sha256_digest_t dest;
2277   calc_hmac_sha256(key, key_len, msg, msg_len,
2278                    reinterpret_cast<char*>(dest.v));
2279   return dest;
2280 }
2281
2282 static inline sha256_digest_t
2283 calc_hmac_sha256(const std::string_view& key, const std::string_view& msg) {
2284   sha256_digest_t dest;
2285   calc_hmac_sha256(key.data(), key.size(),
2286                    msg.data(), msg.size(),
2287                    reinterpret_cast<char*>(dest.v));
2288   return dest;
2289 }
2290
2291 static inline sha256_digest_t
2292 calc_hmac_sha256(const sha256_digest_t &key,
2293                  const std::string_view& msg) {
2294   sha256_digest_t dest;
2295   calc_hmac_sha256(reinterpret_cast<const char*>(key.v), sha256_digest_t::SIZE,
2296                    msg.data(), msg.size(),
2297                    reinterpret_cast<char*>(dest.v));
2298   return dest;
2299 }
2300
2301 static inline sha256_digest_t
2302 calc_hmac_sha256(const std::vector<unsigned char>& key,
2303                  const std::string_view& msg) {
2304   sha256_digest_t dest;
2305   calc_hmac_sha256(reinterpret_cast<const char*>(key.data()), key.size(),
2306                    msg.data(), msg.size(),
2307                    reinterpret_cast<char*>(dest.v));
2308   return dest;
2309 }
2310
2311 template<size_t KeyLenN>
2312 static inline sha256_digest_t
2313 calc_hmac_sha256(const std::array<unsigned char, KeyLenN>& key,
2314                  const std::string_view& msg) {
2315   sha256_digest_t dest;
2316   calc_hmac_sha256(reinterpret_cast<const char*>(key.data()), key.size(),
2317                    msg.data(), msg.size(),
2318                    reinterpret_cast<char*>(dest.v));
2319   return dest;
2320 }
2321
2322 extern sha256_digest_t calc_hash_sha256(const std::string_view& msg);
2323
2324 extern ceph::crypto::SHA256* calc_hash_sha256_open_stream();
2325 extern void calc_hash_sha256_update_stream(ceph::crypto::SHA256* hash,
2326                                            const char* msg,
2327                                            int len);
2328 extern std::string calc_hash_sha256_close_stream(ceph::crypto::SHA256** phash);
2329 extern std::string calc_hash_sha256_restart_stream(ceph::crypto::SHA256** phash);
2330
2331 extern int rgw_parse_op_type_list(const std::string& str, uint32_t *perm);
2332
2333 static constexpr uint32_t MATCH_POLICY_ACTION = 0x01;
2334 static constexpr uint32_t MATCH_POLICY_RESOURCE = 0x02;
2335 static constexpr uint32_t MATCH_POLICY_ARN = 0x04;
2336 static constexpr uint32_t MATCH_POLICY_STRING = 0x08;
2337
2338 extern bool match_policy(std::string_view pattern, std::string_view input,
2339                          uint32_t flag);
2340
2341 extern std::string camelcase_dash_http_attr(const std::string& orig);
2342 extern std::string lowercase_dash_http_attr(const std::string& orig);
2343
2344 void rgw_setup_saved_curl_handles();
2345 void rgw_release_all_curl_handles();
2346
2347 static inline void rgw_escape_str(const std::string& s, char esc_char,
2348                                   char special_char, std::string *dest)
2349 {
2350   const char *src = s.c_str();
2351   char dest_buf[s.size() * 2 + 1];
2352   char *destp = dest_buf;
2353
2354   for (size_t i = 0; i < s.size(); i++) {
2355     char c = src[i];
2356     if (c == esc_char || c == special_char) {
2357       *destp++ = esc_char;
2358     }
2359     *destp++ = c;
2360   }
2361   *destp++ = '\0';
2362   *dest = dest_buf;
2363 }
2364
2365 static inline ssize_t rgw_unescape_str(const std::string& s, ssize_t ofs,
2366                                        char esc_char, char special_char,
2367                                        std::string *dest)
2368 {
2369   const char *src = s.c_str();
2370   char dest_buf[s.size() + 1];
2371   char *destp = dest_buf;
2372   bool esc = false;
2373
2374   dest_buf[0] = '\0';
2375
2376   for (size_t i = ofs; i < s.size(); i++) {
2377     char c = src[i];
2378     if (!esc && c == esc_char) {
2379       esc = true;
2380       continue;
2381     }
2382     if (!esc && c == special_char) {
2383       *destp = '\0';
2384       *dest = dest_buf;
2385       return (ssize_t)i + 1;
2386     }
2387     *destp++ = c;
2388     esc = false;
2389   }
2390   *destp = '\0';
2391   *dest = dest_buf;
2392   return std::string::npos;
2393 }
2394
2395 static inline std::string rgw_bl_str(ceph::buffer::list& raw)
2396 {
2397   size_t len = raw.length();
2398   std::string s(raw.c_str(), len);
2399   while (len && !s[len - 1]) {
2400     --len;
2401     s.resize(len);
2402   }
2403   return s;
2404 }
2405
2406 template <typename T>
2407 int decode_bl(bufferlist& bl, T& t)
2408 {
2409   auto iter = bl.cbegin();
2410   try {
2411     decode(t, iter);
2412   } catch (buffer::error& err) {
2413     return -EIO;
2414   }
2415   return 0;
2416 }