ceph/src/rgw/rgw_common.h

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab ft=cpp
   3
   4 /*
   5  * Ceph - scalable distributed file system
   6  *
   7  * Copyright (C) 2004-2009 Sage Weil <sage@newdream.net>
   8  * Copyright (C) 2015 Yehuda Sadeh <yehuda@redhat.com>
   9  *
  10  * This is free software; you can redistribute it and/or
  11  * modify it under the terms of the GNU Lesser General Public
  12  * License version 2.1, as published by the Free Software
  13  * Foundation.  See file COPYING.
  14  *
  15  */
  16
  17 #ifndef CEPH_RGW_COMMON_H
  18 #define CEPH_RGW_COMMON_H
  19
  20 #include <array>
  21
  22 #include <boost/utility/string_view.hpp>
  23
  24 #include "common/ceph_crypto.h"
  25 #include "common/random_string.h"
  26 #include "rgw_acl.h"
  27 #include "rgw_cors.h"
  28 #include "rgw_iam_policy.h"
  29 #include "rgw_quota.h"
  30 #include "rgw_string.h"
  31 #include "common/async/yield_context.h"
  32 #include "rgw_website.h"
  33 #include "rgw_object_lock.h"
  34 #include "rgw_tag.h"
  35 #include "rgw_sync_policy.h"
  36 #include "cls/version/cls_version_types.h"
  37 #include "cls/user/cls_user_types.h"
  38 #include "cls/rgw/cls_rgw_types.h"
  39 #include "include/rados/librados.hpp"
  40 #include "rgw_public_access.h"
  41
  42 namespace ceph {
  43   class Formatter;
  44 }
  45
  46 namespace rgw::sal {
  47   class RGWUser;
  48 }
  49
  50 using ceph::crypto::MD5;
  51
  52 #define RGW_ATTR_PREFIX  "user.rgw."
  53
  54 #define RGW_HTTP_RGWX_ATTR_PREFIX "RGWX_ATTR_"
  55 #define RGW_HTTP_RGWX_ATTR_PREFIX_OUT "Rgwx-Attr-"
  56
  57 #define RGW_AMZ_PREFIX "x-amz-"
  58 #define RGW_AMZ_META_PREFIX RGW_AMZ_PREFIX "meta-"
  59 #define RGW_AMZ_WEBSITE_REDIRECT_LOCATION RGW_AMZ_PREFIX "website-redirect-location"
  60 #define RGW_AMZ_TAG_COUNT RGW_AMZ_PREFIX "tagging-count"
  61
  62 #define RGW_SYS_PARAM_PREFIX "rgwx-"
  63
  64 #define RGW_ATTR_ACL            RGW_ATTR_PREFIX "acl"
  65 #define RGW_ATTR_LC            RGW_ATTR_PREFIX "lc"
  66 #define RGW_ATTR_CORS           RGW_ATTR_PREFIX "cors"
  67 #define RGW_ATTR_ETAG           RGW_ATTR_PREFIX "etag"
  68 #define RGW_ATTR_BUCKETS        RGW_ATTR_PREFIX "buckets"
  69 #define RGW_ATTR_META_PREFIX    RGW_ATTR_PREFIX RGW_AMZ_META_PREFIX
  70 #define RGW_ATTR_CONTENT_TYPE   RGW_ATTR_PREFIX "content_type"
  71 #define RGW_ATTR_CACHE_CONTROL  RGW_ATTR_PREFIX "cache_control"
  72 #define RGW_ATTR_CONTENT_DISP   RGW_ATTR_PREFIX "content_disposition"
  73 #define RGW_ATTR_CONTENT_ENC    RGW_ATTR_PREFIX "content_encoding"
  74 #define RGW_ATTR_CONTENT_LANG   RGW_ATTR_PREFIX "content_language"
  75 #define RGW_ATTR_EXPIRES        RGW_ATTR_PREFIX "expires"
  76 #define RGW_ATTR_DELETE_AT      RGW_ATTR_PREFIX "delete_at"
  77 #define RGW_ATTR_ID_TAG         RGW_ATTR_PREFIX "idtag"
  78 #define RGW_ATTR_TAIL_TAG       RGW_ATTR_PREFIX "tail_tag"
  79 #define RGW_ATTR_SHADOW_OBJ     RGW_ATTR_PREFIX "shadow_name"
  80 #define RGW_ATTR_MANIFEST       RGW_ATTR_PREFIX "manifest"
  81 #define RGW_ATTR_USER_MANIFEST  RGW_ATTR_PREFIX "user_manifest"
  82 #define RGW_ATTR_AMZ_WEBSITE_REDIRECT_LOCATION  RGW_ATTR_PREFIX RGW_AMZ_WEBSITE_REDIRECT_LOCATION
  83 #define RGW_ATTR_SLO_MANIFEST   RGW_ATTR_PREFIX "slo_manifest"
  84 /* Information whether an object is SLO or not must be exposed to
  85  * user through custom HTTP header named X-Static-Large-Object. */
  86 #define RGW_ATTR_SLO_UINDICATOR RGW_ATTR_META_PREFIX "static-large-object"
  87 #define RGW_ATTR_X_ROBOTS_TAG   RGW_ATTR_PREFIX "x-robots-tag"
  88 #define RGW_ATTR_STORAGE_CLASS  RGW_ATTR_PREFIX "storage_class"
  89
  90 /* S3 Object Lock*/
  91 #define RGW_ATTR_OBJECT_LOCK        RGW_ATTR_PREFIX "object-lock"
  92 #define RGW_ATTR_OBJECT_RETENTION   RGW_ATTR_PREFIX "object-retention"
  93 #define RGW_ATTR_OBJECT_LEGAL_HOLD  RGW_ATTR_PREFIX "object-legal-hold"
  94
  95
  96 #define RGW_ATTR_PG_VER         RGW_ATTR_PREFIX "pg_ver"
  97 #define RGW_ATTR_SOURCE_ZONE    RGW_ATTR_PREFIX "source_zone"
  98 #define RGW_ATTR_TAGS           RGW_ATTR_PREFIX RGW_AMZ_PREFIX "tagging"
  99
 100 #define RGW_ATTR_TEMPURL_KEY1   RGW_ATTR_META_PREFIX "temp-url-key"
 101 #define RGW_ATTR_TEMPURL_KEY2   RGW_ATTR_META_PREFIX "temp-url-key-2"
 102
 103 /* Account/container quota of the Swift API. */
 104 #define RGW_ATTR_QUOTA_NOBJS    RGW_ATTR_META_PREFIX "quota-count"
 105 #define RGW_ATTR_QUOTA_MSIZE    RGW_ATTR_META_PREFIX "quota-bytes"
 106
 107 /* Static Web Site of Swift API. */
 108 #define RGW_ATTR_WEB_INDEX      RGW_ATTR_META_PREFIX "web-index"
 109 #define RGW_ATTR_WEB_ERROR      RGW_ATTR_META_PREFIX "web-error"
 110 #define RGW_ATTR_WEB_LISTINGS   RGW_ATTR_META_PREFIX "web-listings"
 111 #define RGW_ATTR_WEB_LIST_CSS   RGW_ATTR_META_PREFIX "web-listings-css"
 112 #define RGW_ATTR_SUBDIR_MARKER  RGW_ATTR_META_PREFIX "web-directory-type"
 113
 114 #define RGW_ATTR_OLH_PREFIX     RGW_ATTR_PREFIX "olh."
 115
 116 #define RGW_ATTR_OLH_INFO       RGW_ATTR_OLH_PREFIX "info"
 117 #define RGW_ATTR_OLH_VER        RGW_ATTR_OLH_PREFIX "ver"
 118 #define RGW_ATTR_OLH_ID_TAG     RGW_ATTR_OLH_PREFIX "idtag"
 119 #define RGW_ATTR_OLH_PENDING_PREFIX RGW_ATTR_OLH_PREFIX "pending."
 120
 121 #define RGW_ATTR_COMPRESSION    RGW_ATTR_PREFIX "compression"
 122
 123 #define RGW_ATTR_APPEND_PART_NUM    RGW_ATTR_PREFIX "append_part_num"
 124
 125 /* IAM Policy */
 126 #define RGW_ATTR_IAM_POLICY     RGW_ATTR_PREFIX "iam-policy"
 127 #define RGW_ATTR_USER_POLICY    RGW_ATTR_PREFIX "user-policy"
 128 #define RGW_ATTR_PUBLIC_ACCESS  RGW_ATTR_PREFIX "public-access"
 129
 130 /* RGW File Attributes */
 131 #define RGW_ATTR_UNIX_KEY1      RGW_ATTR_PREFIX "unix-key1"
 132 #define RGW_ATTR_UNIX1          RGW_ATTR_PREFIX "unix1"
 133
 134 #define RGW_ATTR_CRYPT_PREFIX   RGW_ATTR_PREFIX "crypt."
 135 #define RGW_ATTR_CRYPT_MODE     RGW_ATTR_CRYPT_PREFIX "mode"
 136 #define RGW_ATTR_CRYPT_KEYMD5   RGW_ATTR_CRYPT_PREFIX "keymd5"
 137 #define RGW_ATTR_CRYPT_KEYID    RGW_ATTR_CRYPT_PREFIX "keyid"
 138 #define RGW_ATTR_CRYPT_KEYSEL   RGW_ATTR_CRYPT_PREFIX "keysel"
 139
 140
 141 #define RGW_FORMAT_PLAIN        0
 142 #define RGW_FORMAT_XML          1
 143 #define RGW_FORMAT_JSON         2
 144 #define RGW_FORMAT_HTML         3
 145
 146 #define RGW_CAP_READ            0x1
 147 #define RGW_CAP_WRITE           0x2
 148 #define RGW_CAP_ALL             (RGW_CAP_READ | RGW_CAP_WRITE)
 149
 150 #define RGW_REST_SWIFT          0x1
 151 #define RGW_REST_SWIFT_AUTH     0x2
 152 #define RGW_REST_S3             0x4
 153 #define RGW_REST_WEBSITE     0x8
 154 #define RGW_REST_STS            0x10
 155 #define RGW_REST_IAM            0x20
 156
 157 #define RGW_SUSPENDED_USER_AUID (uint64_t)-2
 158
 159 #define RGW_OP_TYPE_READ         0x01
 160 #define RGW_OP_TYPE_WRITE        0x02
 161 #define RGW_OP_TYPE_DELETE       0x04
 162
 163 #define RGW_OP_TYPE_MODIFY       (RGW_OP_TYPE_WRITE | RGW_OP_TYPE_DELETE)
 164 #define RGW_OP_TYPE_ALL          (RGW_OP_TYPE_READ | RGW_OP_TYPE_WRITE | RGW_OP_TYPE_DELETE)
 165
 166 #define RGW_DEFAULT_MAX_BUCKETS 1000
 167
 168 #define RGW_DEFER_TO_BUCKET_ACLS_RECURSE 1
 169 #define RGW_DEFER_TO_BUCKET_ACLS_FULL_CONTROL 2
 170
 171 #define STATUS_CREATED           1900
 172 #define STATUS_ACCEPTED          1901
 173 #define STATUS_NO_CONTENT        1902
 174 #define STATUS_PARTIAL_CONTENT   1903
 175 #define STATUS_REDIRECT          1904
 176 #define STATUS_NO_APPLY          1905
 177 #define STATUS_APPLIED           1906
 178
 179 #define ERR_INVALID_BUCKET_NAME  2000
 180 #define ERR_INVALID_OBJECT_NAME  2001
 181 #define ERR_NO_SUCH_BUCKET       2002
 182 #define ERR_METHOD_NOT_ALLOWED   2003
 183 #define ERR_INVALID_DIGEST       2004
 184 #define ERR_BAD_DIGEST           2005
 185 #define ERR_UNRESOLVABLE_EMAIL   2006
 186 #define ERR_INVALID_PART         2007
 187 #define ERR_INVALID_PART_ORDER   2008
 188 #define ERR_NO_SUCH_UPLOAD       2009
 189 #define ERR_REQUEST_TIMEOUT      2010
 190 #define ERR_LENGTH_REQUIRED      2011
 191 #define ERR_REQUEST_TIME_SKEWED  2012
 192 #define ERR_BUCKET_EXISTS        2013
 193 #define ERR_BAD_URL              2014
 194 #define ERR_PRECONDITION_FAILED  2015
 195 #define ERR_NOT_MODIFIED         2016
 196 #define ERR_INVALID_UTF8         2017
 197 #define ERR_UNPROCESSABLE_ENTITY 2018
 198 #define ERR_TOO_LARGE            2019
 199 #define ERR_TOO_MANY_BUCKETS     2020
 200 #define ERR_INVALID_REQUEST      2021
 201 #define ERR_TOO_SMALL            2022
 202 #define ERR_NOT_FOUND            2023
 203 #define ERR_PERMANENT_REDIRECT   2024
 204 #define ERR_LOCKED               2025
 205 #define ERR_QUOTA_EXCEEDED       2026
 206 #define ERR_SIGNATURE_NO_MATCH   2027
 207 #define ERR_INVALID_ACCESS_KEY   2028
 208 #define ERR_MALFORMED_XML        2029
 209 #define ERR_USER_EXIST           2030
 210 #define ERR_NOT_SLO_MANIFEST     2031
 211 #define ERR_EMAIL_EXIST          2032
 212 #define ERR_KEY_EXIST            2033
 213 #define ERR_INVALID_SECRET_KEY   2034
 214 #define ERR_INVALID_KEY_TYPE     2035
 215 #define ERR_INVALID_CAP          2036
 216 #define ERR_INVALID_TENANT_NAME  2037
 217 #define ERR_WEBSITE_REDIRECT     2038
 218 #define ERR_NO_SUCH_WEBSITE_CONFIGURATION 2039
 219 #define ERR_AMZ_CONTENT_SHA256_MISMATCH 2040
 220 #define ERR_NO_SUCH_LC           2041
 221 #define ERR_NO_SUCH_USER         2042
 222 #define ERR_NO_SUCH_SUBUSER      2043
 223 #define ERR_MFA_REQUIRED         2044
 224 #define ERR_NO_SUCH_CORS_CONFIGURATION 2045
 225 #define ERR_NO_SUCH_OBJECT_LOCK_CONFIGURATION  2046
 226 #define ERR_INVALID_RETENTION_PERIOD 2047
 227 #define ERR_USER_SUSPENDED       2100
 228 #define ERR_INTERNAL_ERROR       2200
 229 #define ERR_NOT_IMPLEMENTED      2201
 230 #define ERR_SERVICE_UNAVAILABLE  2202
 231 #define ERR_ROLE_EXISTS          2203
 232 #define ERR_MALFORMED_DOC        2204
 233 #define ERR_NO_ROLE_FOUND        2205
 234 #define ERR_DELETE_CONFLICT      2206
 235 #define ERR_NO_SUCH_BUCKET_POLICY  2207
 236 #define ERR_INVALID_LOCATION_CONSTRAINT 2208
 237 #define ERR_TAG_CONFLICT         2209
 238 #define ERR_INVALID_TAG          2210
 239 #define ERR_ZERO_IN_URL          2211
 240 #define ERR_MALFORMED_ACL_ERROR  2212
 241 #define ERR_ZONEGROUP_DEFAULT_PLACEMENT_MISCONFIGURATION 2213
 242 #define ERR_INVALID_ENCRYPTION_ALGORITHM                 2214
 243 #define ERR_INVALID_CORS_RULES_ERROR                     2215
 244 #define ERR_NO_CORS_FOUND        2216
 245 #define ERR_INVALID_WEBSITE_ROUTING_RULES_ERROR          2217
 246 #define ERR_RATE_LIMITED         2218
 247 #define ERR_POSITION_NOT_EQUAL_TO_LENGTH                 2219
 248 #define ERR_OBJECT_NOT_APPENDABLE                        2220
 249 #define ERR_INVALID_BUCKET_STATE                         2221
 250
 251 #define ERR_BUSY_RESHARDING      2300
 252 #define ERR_NO_SUCH_ENTITY       2301
 253 #define ERR_LIMIT_EXCEEDED       2302
 254
 255 // STS Errors
 256 #define ERR_PACKED_POLICY_TOO_LARGE 2400
 257 #define ERR_INVALID_IDENTITY_TOKEN  2401
 258
 259 #define ERR_NO_SUCH_TAG_SET 2402
 260
 261 #ifndef UINT32_MAX
 262 #define UINT32_MAX (0xffffffffu)
 263 #endif
 264
 265 struct req_state;
 266
 267 typedef void *RGWAccessHandle;
 268
 269 enum RGWIntentEvent {
 270   DEL_OBJ = 0,
 271   DEL_DIR = 1,
 272 };
 273
 274 enum HostStyle {
 275   PathStyle = 0,
 276   VirtualStyle = 1,
 277 };
 278
 279 /** Store error returns for output at a different point in the program */
 280 struct rgw_err {
 281   rgw_err();
 282   void clear();
 283   bool is_clear() const;
 284   bool is_err() const;
 285   friend std::ostream& operator<<(std::ostream& oss, const rgw_err &err);
 286
 287   int http_ret;
 288   int ret;
 289   std::string err_code;
 290   std::string message;
 291 };
 292
 293 /* Helper class used for RGWHTTPArgs parsing */
 294 class NameVal
 295 {
 296    const std::string str;
 297    std::string name;
 298    std::string val;
 299  public:
 300     explicit NameVal(const std::string& nv) : str(nv) {}
 301
 302     int parse();
 303
 304     std::string& get_name() { return name; }
 305     std::string& get_val() { return val; }
 306 };
 307
 308 /** Stores the XML arguments associated with the HTTP request in req_state*/
 309 class RGWHTTPArgs {
 310   std::string str, empty_str;
 311   std::map<std::string, std::string> val_map;
 312   std::map<std::string, std::string> sys_val_map;
 313   std::map<std::string, std::string> sub_resources;
 314   bool has_resp_modifier = false;
 315   bool admin_subresource_added = false;
 316  public:
 317   RGWHTTPArgs() = default;
 318   explicit RGWHTTPArgs(const std::string& s) {
 319       set(s);
 320       parse();
 321   }
 322
 323   /** Set the arguments; as received */
 324   void set(const std::string& s) {
 325     has_resp_modifier = false;
 326     val_map.clear();
 327     sub_resources.clear();
 328     str = s;
 329   }
 330   /** parse the received arguments */
 331   int parse();
 332   void append(const std::string& name, const string& val);
 333   /** Get the value for a specific argument parameter */
 334   const string& get(const std::string& name, bool *exists = NULL) const;
 335   boost::optional<const std::string&>
 336   get_optional(const std::string& name) const;
 337   int get_bool(const std::string& name, bool *val, bool *exists);
 338   int get_bool(const char *name, bool *val, bool *exists);
 339   void get_bool(const char *name, bool *val, bool def_val);
 340   int get_int(const char *name, int *val, int def_val);
 341
 342   /** Get the value for specific system argument parameter */
 343   std::string sys_get(const std::string& name, bool *exists = nullptr) const;
 344
 345   /** see if a parameter is contained in this RGWHTTPArgs */
 346   bool exists(const char *name) const {
 347     return (val_map.find(name) != std::end(val_map));
 348   }
 349   bool sub_resource_exists(const char *name) const {
 350     return (sub_resources.find(name) != std::end(sub_resources));
 351   }
 352   std::map<std::string, std::string>& get_params() {
 353     return val_map;
 354   }
 355   const std::map<std::string, std::string>& get_sub_resources() const {
 356     return sub_resources;
 357   }
 358   unsigned get_num_params() const {
 359     return val_map.size();
 360   }
 361   bool has_response_modifier() const {
 362     return has_resp_modifier;
 363   }
 364   void set_system() { /* make all system params visible */
 365     std::map<std::string, std::string>::iterator iter;
 366     for (iter = sys_val_map.begin(); iter != sys_val_map.end(); ++iter) {
 367       val_map[iter->first] = iter->second;
 368     }
 369   }
 370   const std::string& get_str() {
 371     return str;
 372   }
 373 }; // RGWHTTPArgs
 374
 375 const char *rgw_conf_get(const map<string, string, ltstr_nocase>& conf_map, const char *name, const char *def_val);
 376 int rgw_conf_get_int(const map<string, string, ltstr_nocase>& conf_map, const char *name, int def_val);
 377 bool rgw_conf_get_bool(const map<string, string, ltstr_nocase>& conf_map, const char *name, bool def_val);
 378
 379 class RGWEnv;
 380
 381 class RGWConf {
 382   friend class RGWEnv;
 383   int enable_ops_log;
 384   int enable_usage_log;
 385   uint8_t defer_to_bucket_acls;
 386   void init(CephContext *cct);
 387 public:
 388   RGWConf()
 389     : enable_ops_log(1),
 390       enable_usage_log(1),
 391       defer_to_bucket_acls(0) {
 392   }
 393 };
 394
 395 class RGWEnv {
 396   std::map<string, string, ltstr_nocase> env_map;
 397   RGWConf conf;
 398 public:
 399   void init(CephContext *cct);
 400   void init(CephContext *cct, char **envp);
 401   void set(std::string name, std::string val);
 402   const char *get(const char *name, const char *def_val = nullptr) const;
 403   int get_int(const char *name, int def_val = 0) const;
 404   bool get_bool(const char *name, bool def_val = 0);
 405   size_t get_size(const char *name, size_t def_val = 0) const;
 406   bool exists(const char *name) const;
 407   bool exists_prefix(const char *prefix) const;
 408   void remove(const char *name);
 409   const std::map<string, string, ltstr_nocase>& get_map() const { return env_map; }
 410   int get_enable_ops_log() const {
 411     return conf.enable_ops_log;
 412   }
 413
 414   int get_enable_usage_log() const {
 415     return conf.enable_usage_log;
 416   }
 417
 418   int get_defer_to_bucket_acls() const {
 419     return conf.defer_to_bucket_acls;
 420   }
 421 };
 422
 423 // return true if the connection is secure. this either means that the
 424 // connection arrived via ssl, or was forwarded as https by a trusted proxy
 425 bool rgw_transport_is_secure(CephContext *cct, const RGWEnv& env);
 426
 427 enum http_op {
 428   OP_GET,
 429   OP_PUT,
 430   OP_DELETE,
 431   OP_HEAD,
 432   OP_POST,
 433   OP_COPY,
 434   OP_OPTIONS,
 435   OP_UNKNOWN,
 436 };
 437
 438 enum RGWOpType {
 439   RGW_OP_UNKNOWN = 0,
 440   RGW_OP_GET_OBJ,
 441   RGW_OP_LIST_BUCKETS,
 442   RGW_OP_STAT_ACCOUNT,
 443   RGW_OP_LIST_BUCKET,
 444   RGW_OP_GET_BUCKET_LOGGING,
 445   RGW_OP_GET_BUCKET_LOCATION,
 446   RGW_OP_GET_BUCKET_VERSIONING,
 447   RGW_OP_SET_BUCKET_VERSIONING,
 448   RGW_OP_GET_BUCKET_WEBSITE,
 449   RGW_OP_SET_BUCKET_WEBSITE,
 450   RGW_OP_STAT_BUCKET,
 451   RGW_OP_CREATE_BUCKET,
 452   RGW_OP_DELETE_BUCKET,
 453   RGW_OP_PUT_OBJ,
 454   RGW_OP_STAT_OBJ,
 455   RGW_OP_POST_OBJ,
 456   RGW_OP_PUT_METADATA_ACCOUNT,
 457   RGW_OP_PUT_METADATA_BUCKET,
 458   RGW_OP_PUT_METADATA_OBJECT,
 459   RGW_OP_SET_TEMPURL,
 460   RGW_OP_DELETE_OBJ,
 461   RGW_OP_COPY_OBJ,
 462   RGW_OP_GET_ACLS,
 463   RGW_OP_PUT_ACLS,
 464   RGW_OP_GET_CORS,
 465   RGW_OP_PUT_CORS,
 466   RGW_OP_DELETE_CORS,
 467   RGW_OP_OPTIONS_CORS,
 468   RGW_OP_GET_REQUEST_PAYMENT,
 469   RGW_OP_SET_REQUEST_PAYMENT,
 470   RGW_OP_INIT_MULTIPART,
 471   RGW_OP_COMPLETE_MULTIPART,
 472   RGW_OP_ABORT_MULTIPART,
 473   RGW_OP_LIST_MULTIPART,
 474   RGW_OP_LIST_BUCKET_MULTIPARTS,
 475   RGW_OP_DELETE_MULTI_OBJ,
 476   RGW_OP_BULK_DELETE,
 477   RGW_OP_SET_ATTRS,
 478   RGW_OP_GET_CROSS_DOMAIN_POLICY,
 479   RGW_OP_GET_HEALTH_CHECK,
 480   RGW_OP_GET_INFO,
 481   RGW_OP_CREATE_ROLE,
 482   RGW_OP_DELETE_ROLE,
 483   RGW_OP_GET_ROLE,
 484   RGW_OP_MODIFY_ROLE,
 485   RGW_OP_LIST_ROLES,
 486   RGW_OP_PUT_ROLE_POLICY,
 487   RGW_OP_GET_ROLE_POLICY,
 488   RGW_OP_LIST_ROLE_POLICIES,
 489   RGW_OP_DELETE_ROLE_POLICY,
 490   RGW_OP_PUT_BUCKET_POLICY,
 491   RGW_OP_GET_BUCKET_POLICY,
 492   RGW_OP_DELETE_BUCKET_POLICY,
 493   RGW_OP_PUT_OBJ_TAGGING,
 494   RGW_OP_GET_OBJ_TAGGING,
 495   RGW_OP_DELETE_OBJ_TAGGING,
 496   RGW_OP_PUT_LC,
 497   RGW_OP_GET_LC,
 498   RGW_OP_DELETE_LC,
 499   RGW_OP_PUT_USER_POLICY,
 500   RGW_OP_GET_USER_POLICY,
 501   RGW_OP_LIST_USER_POLICIES,
 502   RGW_OP_DELETE_USER_POLICY,
 503   RGW_OP_PUT_BUCKET_OBJ_LOCK,
 504   RGW_OP_GET_BUCKET_OBJ_LOCK,
 505   RGW_OP_PUT_OBJ_RETENTION,
 506   RGW_OP_GET_OBJ_RETENTION,
 507   RGW_OP_PUT_OBJ_LEGAL_HOLD,
 508   RGW_OP_GET_OBJ_LEGAL_HOLD,
 509   /* rgw specific */
 510   RGW_OP_ADMIN_SET_METADATA,
 511   RGW_OP_GET_OBJ_LAYOUT,
 512   RGW_OP_BULK_UPLOAD,
 513   RGW_OP_METADATA_SEARCH,
 514   RGW_OP_CONFIG_BUCKET_META_SEARCH,
 515   RGW_OP_GET_BUCKET_META_SEARCH,
 516   RGW_OP_DEL_BUCKET_META_SEARCH,
 517   /* sts specific*/
 518   RGW_STS_ASSUME_ROLE,
 519   RGW_STS_GET_SESSION_TOKEN,
 520   RGW_STS_ASSUME_ROLE_WEB_IDENTITY,
 521   /* pubsub */
 522   RGW_OP_PUBSUB_TOPIC_CREATE,
 523   RGW_OP_PUBSUB_TOPICS_LIST,
 524   RGW_OP_PUBSUB_TOPIC_GET,
 525   RGW_OP_PUBSUB_TOPIC_DELETE,
 526   RGW_OP_PUBSUB_SUB_CREATE,
 527   RGW_OP_PUBSUB_SUB_GET,
 528   RGW_OP_PUBSUB_SUB_DELETE,
 529   RGW_OP_PUBSUB_SUB_PULL,
 530   RGW_OP_PUBSUB_SUB_ACK,
 531   RGW_OP_PUBSUB_NOTIF_CREATE,
 532   RGW_OP_PUBSUB_NOTIF_DELETE,
 533   RGW_OP_PUBSUB_NOTIF_LIST,
 534   RGW_OP_GET_BUCKET_TAGGING,
 535   RGW_OP_PUT_BUCKET_TAGGING,
 536   RGW_OP_DELETE_BUCKET_TAGGING,
 537   RGW_OP_GET_BUCKET_REPLICATION,
 538   RGW_OP_PUT_BUCKET_REPLICATION,
 539   RGW_OP_DELETE_BUCKET_REPLICATION,
 540
 541   /* public access */
 542   RGW_OP_GET_BUCKET_POLICY_STATUS,
 543   RGW_OP_PUT_BUCKET_PUBLIC_ACCESS_BLOCK,
 544   RGW_OP_GET_BUCKET_PUBLIC_ACCESS_BLOCK,
 545   RGW_OP_DELETE_BUCKET_PUBLIC_ACCESS_BLOCK,
 546
 547   /*OIDC provider specific*/
 548   RGW_OP_CREATE_OIDC_PROVIDER,
 549   RGW_OP_DELETE_OIDC_PROVIDER,
 550   RGW_OP_GET_OIDC_PROVIDER,
 551   RGW_OP_LIST_OIDC_PROVIDERS,
 552 };
 553
 554 class RGWAccessControlPolicy;
 555 class JSONObj;
 556
 557 struct RGWAccessKey {
 558   string id; // AccessKey
 559   string key; // SecretKey
 560   string subuser;
 561
 562   RGWAccessKey() {}
 563   RGWAccessKey(std::string _id, std::string _key)
 564     : id(std::move(_id)), key(std::move(_key)) {}
 565
 566   void encode(bufferlist& bl) const {
 567     ENCODE_START(2, 2, bl);
 568     encode(id, bl);
 569     encode(key, bl);
 570     encode(subuser, bl);
 571     ENCODE_FINISH(bl);
 572   }
 573
 574   void decode(bufferlist::const_iterator& bl) {
 575      DECODE_START_LEGACY_COMPAT_LEN_32(2, 2, 2, bl);
 576      decode(id, bl);
 577      decode(key, bl);
 578      decode(subuser, bl);
 579      DECODE_FINISH(bl);
 580   }
 581   void dump(Formatter *f) const;
 582   void dump_plain(Formatter *f) const;
 583   void dump(Formatter *f, const string& user, bool swift) const;
 584   static void generate_test_instances(list<RGWAccessKey*>& o);
 585
 586   void decode_json(JSONObj *obj);
 587   void decode_json(JSONObj *obj, bool swift);
 588 };
 589 WRITE_CLASS_ENCODER(RGWAccessKey)
 590
 591 struct RGWSubUser {
 592   string name;
 593   uint32_t perm_mask;
 594
 595   RGWSubUser() : perm_mask(0) {}
 596   void encode(bufferlist& bl) const {
 597     ENCODE_START(2, 2, bl);
 598     encode(name, bl);
 599     encode(perm_mask, bl);
 600     ENCODE_FINISH(bl);
 601   }
 602
 603   void decode(bufferlist::const_iterator& bl) {
 604      DECODE_START_LEGACY_COMPAT_LEN_32(2, 2, 2, bl);
 605      decode(name, bl);
 606      decode(perm_mask, bl);
 607      DECODE_FINISH(bl);
 608   }
 609   void dump(Formatter *f) const;
 610   void dump(Formatter *f, const string& user) const;
 611   static void generate_test_instances(list<RGWSubUser*>& o);
 612
 613   void decode_json(JSONObj *obj);
 614 };
 615 WRITE_CLASS_ENCODER(RGWSubUser)
 616
 617 class RGWUserCaps
 618 {
 619   map<string, uint32_t> caps;
 620
 621   int get_cap(const string& cap, string& type, uint32_t *perm);
 622   int add_cap(const string& cap);
 623   int remove_cap(const string& cap);
 624 public:
 625   static int parse_cap_perm(const string& str, uint32_t *perm);
 626   int add_from_string(const string& str);
 627   int remove_from_string(const string& str);
 628
 629   void encode(bufferlist& bl) const {
 630      ENCODE_START(1, 1, bl);
 631      encode(caps, bl);
 632      ENCODE_FINISH(bl);
 633   }
 634   void decode(bufferlist::const_iterator& bl) {
 635      DECODE_START(1, bl);
 636      decode(caps, bl);
 637      DECODE_FINISH(bl);
 638   }
 639   int check_cap(const string& cap, uint32_t perm) const;
 640   bool is_valid_cap_type(const string& tp);
 641   void dump(Formatter *f) const;
 642   void dump(Formatter *f, const char *name) const;
 643
 644   void decode_json(JSONObj *obj);
 645 };
 646 WRITE_CLASS_ENCODER(RGWUserCaps)
 647
 648 void encode_json(const char *name, const obj_version& v, Formatter *f);
 649 void encode_json(const char *name, const RGWUserCaps& val, Formatter *f);
 650
 651 void decode_json_obj(obj_version& v, JSONObj *obj);
 652
 653
 654
 655 enum RGWIdentityType
 656 {
 657   TYPE_NONE=0,
 658   TYPE_RGW=1,
 659   TYPE_KEYSTONE=2,
 660   TYPE_LDAP=3,
 661   TYPE_ROLE=4,
 662   TYPE_WEB=5,
 663 };
 664
 665 static string RGW_STORAGE_CLASS_STANDARD = "STANDARD";
 666
 667 struct rgw_placement_rule {
 668   std::string name;
 669   std::string storage_class;
 670
 671   rgw_placement_rule() {}
 672   rgw_placement_rule(const string& _n, const string& _sc) : name(_n), storage_class(_sc) {}
 673   rgw_placement_rule(const rgw_placement_rule& _r, const string& _sc) : name(_r.name) {
 674     if (!_sc.empty()) {
 675       storage_class = _sc;
 676     } else {
 677       storage_class = _r.storage_class;
 678     }
 679   }
 680
 681   bool empty() const {
 682     return name.empty() && storage_class.empty();
 683   }
 684
 685   void inherit_from(const rgw_placement_rule& r) {
 686     if (name.empty()) {
 687       name = r.name;
 688     }
 689     if (storage_class.empty()) {
 690       storage_class = r.storage_class;
 691     }
 692   }
 693
 694   void clear() {
 695     name.clear();
 696     storage_class.clear();
 697   }
 698
 699   void init(const string& n, const string& c) {
 700     name = n;
 701     storage_class = c;
 702   }
 703
 704   static const string& get_canonical_storage_class(const string& storage_class) {
 705     if (storage_class.empty()) {
 706       return RGW_STORAGE_CLASS_STANDARD;
 707     }
 708     return storage_class;
 709   }
 710
 711   const string& get_storage_class() const {
 712     return get_canonical_storage_class(storage_class);
 713   }
 714
 715   int compare(const rgw_placement_rule& r) const {
 716     int c = name.compare(r.name);
 717     if (c != 0) {
 718       return c;
 719     }
 720     return get_storage_class().compare(r.get_storage_class());
 721   }
 722
 723   bool operator==(const rgw_placement_rule& r) const {
 724     return (name == r.name &&
 725             get_storage_class() == r.get_storage_class());
 726   }
 727
 728   bool operator!=(const rgw_placement_rule& r) const {
 729     return !(*this == r);
 730   }
 731
 732   void encode(bufferlist& bl) const {
 733     /* no ENCODE_START/END due to backward compatibility */
 734     std::string s = to_str();
 735     ceph::encode(s, bl);
 736   }
 737
 738   void decode(bufferlist::const_iterator& bl) {
 739     std::string s;
 740     ceph::decode(s, bl);
 741     from_str(s);
 742   }
 743
 744   std::string to_str() const {
 745     if (standard_storage_class()) {
 746       return name;
 747     }
 748     return to_str_explicit();
 749   }
 750
 751   std::string to_str_explicit() const {
 752     return name + "/" + storage_class;
 753   }
 754
 755   void from_str(const std::string& s) {
 756     size_t pos = s.find("/");
 757     if (pos == std::string::npos) {
 758       name = s;
 759       storage_class.clear();
 760       return;
 761     }
 762     name = s.substr(0, pos);
 763     storage_class = s.substr(pos + 1);
 764   }
 765
 766   bool standard_storage_class() const {
 767     return storage_class.empty() || storage_class == RGW_STORAGE_CLASS_STANDARD;
 768   }
 769 };
 770 WRITE_CLASS_ENCODER(rgw_placement_rule)
 771
 772 void encode_json(const char *name, const rgw_placement_rule& val, ceph::Formatter *f);
 773 void decode_json_obj(rgw_placement_rule& v, JSONObj *obj);
 774
 775 inline ostream& operator<<(ostream& out, const rgw_placement_rule& rule) {
 776   return out << rule.to_str();
 777 }
 778 struct RGWUserInfo
 779 {
 780   rgw_user user_id;
 781   string display_name;
 782   string user_email;
 783   map<string, RGWAccessKey> access_keys;
 784   map<string, RGWAccessKey> swift_keys;
 785   map<string, RGWSubUser> subusers;
 786   __u8 suspended;
 787   int32_t max_buckets;
 788   uint32_t op_mask;
 789   RGWUserCaps caps;
 790   __u8 admin;
 791   __u8 system;
 792   rgw_placement_rule default_placement;
 793   list<string> placement_tags;
 794   RGWQuotaInfo bucket_quota;
 795   map<int, string> temp_url_keys;
 796   RGWQuotaInfo user_quota;
 797   uint32_t type;
 798   set<string> mfa_ids;
 799   string assumed_role_arn;
 800
 801   RGWUserInfo()
 802     : suspended(0),
 803       max_buckets(RGW_DEFAULT_MAX_BUCKETS),
 804       op_mask(RGW_OP_TYPE_ALL),
 805       admin(0),
 806       system(0),
 807       type(TYPE_NONE) {
 808   }
 809
 810   RGWAccessKey* get_key(const string& access_key) {
 811     if (access_keys.empty())
 812       return nullptr;
 813
 814     auto k = access_keys.find(access_key);
 815     if (k == access_keys.end())
 816       return nullptr;
 817     else
 818       return &(k->second);
 819   }
 820
 821   void encode(bufferlist& bl) const {
 822      ENCODE_START(21, 9, bl);
 823      encode((uint64_t)0, bl); // old auid
 824      string access_key;
 825      string secret_key;
 826      if (!access_keys.empty()) {
 827        map<string, RGWAccessKey>::const_iterator iter = access_keys.begin();
 828        const RGWAccessKey& k = iter->second;
 829        access_key = k.id;
 830        secret_key = k.key;
 831      }
 832      encode(access_key, bl);
 833      encode(secret_key, bl);
 834      encode(display_name, bl);
 835      encode(user_email, bl);
 836      string swift_name;
 837      string swift_key;
 838      if (!swift_keys.empty()) {
 839        map<string, RGWAccessKey>::const_iterator iter = swift_keys.begin();
 840        const RGWAccessKey& k = iter->second;
 841        swift_name = k.id;
 842        swift_key = k.key;
 843      }
 844      encode(swift_name, bl);
 845      encode(swift_key, bl);
 846      encode(user_id.id, bl);
 847      encode(access_keys, bl);
 848      encode(subusers, bl);
 849      encode(suspended, bl);
 850      encode(swift_keys, bl);
 851      encode(max_buckets, bl);
 852      encode(caps, bl);
 853      encode(op_mask, bl);
 854      encode(system, bl);
 855      encode(default_placement, bl);
 856      encode(placement_tags, bl);
 857      encode(bucket_quota, bl);
 858      encode(temp_url_keys, bl);
 859      encode(user_quota, bl);
 860      encode(user_id.tenant, bl);
 861      encode(admin, bl);
 862      encode(type, bl);
 863      encode(mfa_ids, bl);
 864      encode(assumed_role_arn, bl);
 865      ENCODE_FINISH(bl);
 866   }
 867   void decode(bufferlist::const_iterator& bl) {
 868      DECODE_START_LEGACY_COMPAT_LEN_32(21, 9, 9, bl);
 869      if (struct_v >= 2) {
 870        uint64_t old_auid;
 871        decode(old_auid, bl);
 872      }
 873      string access_key;
 874      string secret_key;
 875     decode(access_key, bl);
 876     decode(secret_key, bl);
 877     if (struct_v < 6) {
 878       RGWAccessKey k;
 879       k.id = access_key;
 880       k.key = secret_key;
 881       access_keys[access_key] = k;
 882     }
 883     decode(display_name, bl);
 884     decode(user_email, bl);
 885     /* We populate swift_keys map later nowadays, but we have to decode. */
 886     string swift_name;
 887     string swift_key;
 888     if (struct_v >= 3) decode(swift_name, bl);
 889     if (struct_v >= 4) decode(swift_key, bl);
 890     if (struct_v >= 5)
 891       decode(user_id.id, bl);
 892     else
 893       user_id.id = access_key;
 894     if (struct_v >= 6) {
 895       decode(access_keys, bl);
 896       decode(subusers, bl);
 897     }
 898     suspended = 0;
 899     if (struct_v >= 7) {
 900       decode(suspended, bl);
 901     }
 902     if (struct_v >= 8) {
 903       decode(swift_keys, bl);
 904     }
 905     if (struct_v >= 10) {
 906       decode(max_buckets, bl);
 907     } else {
 908       max_buckets = RGW_DEFAULT_MAX_BUCKETS;
 909     }
 910     if (struct_v >= 11) {
 911       decode(caps, bl);
 912     }
 913     if (struct_v >= 12) {
 914       decode(op_mask, bl);
 915     } else {
 916       op_mask = RGW_OP_TYPE_ALL;
 917     }
 918     if (struct_v >= 13) {
 919       decode(system, bl);
 920       decode(default_placement, bl);
 921       decode(placement_tags, bl); /* tags of allowed placement rules */
 922     }
 923     if (struct_v >= 14) {
 924       decode(bucket_quota, bl);
 925     }
 926     if (struct_v >= 15) {
 927      decode(temp_url_keys, bl);
 928     }
 929     if (struct_v >= 16) {
 930       decode(user_quota, bl);
 931     }
 932     if (struct_v >= 17) {
 933       decode(user_id.tenant, bl);
 934     } else {
 935       user_id.tenant.clear();
 936     }
 937     if (struct_v >= 18) {
 938       decode(admin, bl);
 939     }
 940     if (struct_v >= 19) {
 941       decode(type, bl);
 942     }
 943     if (struct_v >= 20) {
 944       decode(mfa_ids, bl);
 945     }
 946     if (struct_v >= 21) {
 947       decode(assumed_role_arn, bl);
 948     }
 949     DECODE_FINISH(bl);
 950   }
 951   void dump(Formatter *f) const;
 952   static void generate_test_instances(list<RGWUserInfo*>& o);
 953
 954   void decode_json(JSONObj *obj);
 955 };
 956 WRITE_CLASS_ENCODER(RGWUserInfo)
 957
 958 struct rgw_raw_obj {
 959   rgw_pool pool;
 960   std::string oid;
 961   std::string loc;
 962
 963   rgw_raw_obj() {}
 964   rgw_raw_obj(const rgw_pool& _pool, const std::string& _oid) {
 965     init(_pool, _oid);
 966   }
 967   rgw_raw_obj(const rgw_pool& _pool, const std::string& _oid, const string& _loc) : loc(_loc) {
 968     init(_pool, _oid);
 969   }
 970
 971   void init(const rgw_pool& _pool, const std::string& _oid) {
 972     pool = _pool;
 973     oid = _oid;
 974   }
 975
 976   bool empty() const {
 977     return oid.empty();
 978   }
 979
 980   void encode(bufferlist& bl) const {
 981      ENCODE_START(6, 6, bl);
 982     encode(pool, bl);
 983     encode(oid, bl);
 984     encode(loc, bl);
 985     ENCODE_FINISH(bl);
 986   }
 987
 988   void decode_from_rgw_obj(bufferlist::const_iterator& bl);
 989
 990   void decode(bufferlist::const_iterator& bl) {
 991     unsigned ofs = bl.get_off();
 992     DECODE_START(6, bl);
 993     if (struct_v < 6) {
 994       /*
 995        * this object was encoded as rgw_obj, prior to rgw_raw_obj been split out of it,
 996        * let's decode it as rgw_obj and convert it
 997        */
 998       bl.seek(ofs);
 999       decode_from_rgw_obj(bl);
1000       return;
1001     }
1002     decode(pool, bl);
1003     decode(oid, bl);
1004     decode(loc, bl);
1005     DECODE_FINISH(bl);
1006   }
1007
1008   bool operator<(const rgw_raw_obj& o) const {
1009     int r = pool.compare(o.pool);
1010     if (r == 0) {
1011       r = oid.compare(o.oid);
1012       if (r == 0) {
1013         r = loc.compare(o.loc);
1014       }
1015     }
1016     return (r < 0);
1017   }
1018
1019   bool operator==(const rgw_raw_obj& o) const {
1020     return (pool == o.pool && oid == o.oid && loc == o.loc);
1021   }
1022
1023   void dump(Formatter *f) const;
1024   void decode_json(JSONObj *obj);
1025 };
1026 WRITE_CLASS_ENCODER(rgw_raw_obj)
1027
1028 inline ostream& operator<<(ostream& out, const rgw_raw_obj& o) {
1029   out << o.pool << ":" << o.oid;
1030   return out;
1031 }
1032
1033 struct rgw_bucket_placement {
1034   rgw_placement_rule placement_rule;
1035   rgw_bucket bucket;
1036
1037   void dump(Formatter *f) const;
1038 };
1039
1040 struct RGWObjVersionTracker {
1041   obj_version read_version;
1042   obj_version write_version;
1043
1044   obj_version *version_for_read() {
1045     return &read_version;
1046   }
1047
1048   obj_version *version_for_write() {
1049     if (write_version.ver == 0)
1050       return NULL;
1051
1052     return &write_version;
1053   }
1054
1055   obj_version *version_for_check() {
1056     if (read_version.ver == 0)
1057       return NULL;
1058
1059     return &read_version;
1060   }
1061
1062   void prepare_op_for_read(librados::ObjectReadOperation *op);
1063   void prepare_op_for_write(librados::ObjectWriteOperation *op);
1064
1065   void apply_write();
1066
1067   void clear() {
1068     read_version = obj_version();
1069     write_version = obj_version();
1070   }
1071
1072   void generate_new_write_ver(CephContext *cct);
1073 };
1074
1075 inline ostream& operator<<(ostream& out, const obj_version &v)
1076 {
1077   out << v.tag << ":" << v.ver;
1078   return out;
1079 }
1080
1081 inline ostream& operator<<(ostream& out, const RGWObjVersionTracker &ot)
1082 {
1083   out << "{r=" << ot.read_version << ",w=" << ot.write_version << "}";
1084   return out;
1085 }
1086
1087 enum RGWBucketFlags {
1088   BUCKET_SUSPENDED = 0x1,
1089   BUCKET_VERSIONED = 0x2,
1090   BUCKET_VERSIONS_SUSPENDED = 0x4,
1091   BUCKET_DATASYNC_DISABLED = 0X8,
1092   BUCKET_MFA_ENABLED = 0X10,
1093   BUCKET_OBJ_LOCK_ENABLED = 0X20,
1094 };
1095
1096 enum RGWBucketIndexType {
1097   RGWBIType_Normal = 0,
1098   RGWBIType_Indexless = 1,
1099 };
1100
1101 inline ostream& operator<<(ostream& out, const RGWBucketIndexType &index_type)
1102 {
1103   switch (index_type) {
1104     case RGWBIType_Normal:
1105       return out << "Normal";
1106     case RGWBIType_Indexless:
1107       return out << "Indexless";
1108     default:
1109       return out << "Unknown";
1110   }
1111 }
1112
1113 class RGWSI_Zone;
1114
1115 struct RGWBucketInfo {
1116   enum BIShardsHashType {
1117     MOD = 0
1118   };
1119
1120   rgw_bucket bucket;
1121   rgw_user owner;
1122   uint32_t flags{0};
1123   string zonegroup;
1124   ceph::real_time creation_time;
1125   rgw_placement_rule placement_rule;
1126   bool has_instance_obj{false};
1127   RGWObjVersionTracker objv_tracker; /* we don't need to serialize this, for runtime tracking */
1128   RGWQuotaInfo quota;
1129
1130   // Represents the number of bucket index object shards:
1131   //   - value of 0 indicates there is no sharding (this is by default
1132   //     before this feature is implemented).
1133   //   - value of UINT32_T::MAX indicates this is a blind bucket.
1134   uint32_t num_shards{0};
1135
1136   // Represents the bucket index shard hash type.
1137   uint8_t bucket_index_shard_hash_type{MOD};
1138
1139   // Represents the shard number for blind bucket.
1140   const static uint32_t NUM_SHARDS_BLIND_BUCKET;
1141
1142   bool requester_pays{false};
1143
1144   bool has_website{false};
1145   RGWBucketWebsiteConf website_conf;
1146
1147   RGWBucketIndexType index_type = RGWBIType_Normal;
1148
1149   bool swift_versioning{false};
1150   string swift_ver_location;
1151
1152   map<string, uint32_t> mdsearch_config;
1153
1154   // resharding
1155   cls_rgw_reshard_status reshard_status{cls_rgw_reshard_status::NOT_RESHARDING};
1156   string new_bucket_instance_id;
1157
1158   RGWObjectLock obj_lock;
1159
1160   std::optional<rgw_sync_policy_info> sync_policy;
1161
1162   void encode(bufferlist& bl) const;
1163   void decode(bufferlist::const_iterator& bl);
1164
1165   void dump(Formatter *f) const;
1166   static void generate_test_instances(list<RGWBucketInfo*>& o);
1167
1168   void decode_json(JSONObj *obj);
1169
1170   bool versioned() const { return (flags & BUCKET_VERSIONED) != 0; }
1171   int versioning_status() const { return flags & (BUCKET_VERSIONED | BUCKET_VERSIONS_SUSPENDED | BUCKET_MFA_ENABLED); }
1172   bool versioning_enabled() const { return (versioning_status() & (BUCKET_VERSIONED | BUCKET_VERSIONS_SUSPENDED)) == BUCKET_VERSIONED; }
1173   bool mfa_enabled() const { return (versioning_status() & BUCKET_MFA_ENABLED) != 0; }
1174   bool datasync_flag_enabled() const { return (flags & BUCKET_DATASYNC_DISABLED) == 0; }
1175   bool obj_lock_enabled() const { return (flags & BUCKET_OBJ_LOCK_ENABLED) != 0; }
1176
1177   bool has_swift_versioning() const {
1178     /* A bucket may be versioned through one mechanism only. */
1179     return swift_versioning && !versioned();
1180   }
1181
1182   void set_sync_policy(rgw_sync_policy_info&& policy);
1183
1184   bool empty_sync_policy() const;
1185
1186   RGWBucketInfo();
1187   ~RGWBucketInfo();
1188 };
1189 WRITE_CLASS_ENCODER(RGWBucketInfo)
1190
1191 struct RGWBucketEntryPoint
1192 {
1193   rgw_bucket bucket;
1194   rgw_user owner;
1195   ceph::real_time creation_time;
1196   bool linked;
1197
1198   bool has_bucket_info;
1199   RGWBucketInfo old_bucket_info;
1200
1201   RGWBucketEntryPoint() : linked(false), has_bucket_info(false) {}
1202
1203   void encode(bufferlist& bl) const {
1204     ENCODE_START(10, 8, bl);
1205     encode(bucket, bl);
1206     encode(owner.id, bl);
1207     encode(linked, bl);
1208     uint64_t ctime = (uint64_t)real_clock::to_time_t(creation_time);
1209     encode(ctime, bl);
1210     encode(owner, bl);
1211     encode(creation_time, bl);
1212     ENCODE_FINISH(bl);
1213   }
1214   void decode(bufferlist::const_iterator& bl) {
1215     auto orig_iter = bl;
1216     DECODE_START_LEGACY_COMPAT_LEN_32(10, 4, 4, bl);
1217     if (struct_v < 8) {
1218       /* ouch, old entry, contains the bucket info itself */
1219       old_bucket_info.decode(orig_iter);
1220       has_bucket_info = true;
1221       return;
1222     }
1223     has_bucket_info = false;
1224     decode(bucket, bl);
1225     decode(owner.id, bl);
1226     decode(linked, bl);
1227     uint64_t ctime;
1228     decode(ctime, bl);
1229     if (struct_v < 10) {
1230       creation_time = real_clock::from_time_t((time_t)ctime);
1231     }
1232     if (struct_v >= 9) {
1233       decode(owner, bl);
1234     }
1235     if (struct_v >= 10) {
1236       decode(creation_time, bl);
1237     }
1238     DECODE_FINISH(bl);
1239   }
1240
1241   void dump(Formatter *f) const;
1242   void decode_json(JSONObj *obj);
1243   static void generate_test_instances(list<RGWBucketEntryPoint*>& o);
1244 };
1245 WRITE_CLASS_ENCODER(RGWBucketEntryPoint)
1246
1247 struct RGWStorageStats
1248 {
1249   RGWObjCategory category;
1250   uint64_t size;
1251   uint64_t size_rounded;
1252   uint64_t size_utilized{0}; //< size after compression, encryption
1253   uint64_t num_objects;
1254
1255   RGWStorageStats()
1256     : category(RGWObjCategory::None),
1257       size(0),
1258       size_rounded(0),
1259       num_objects(0) {}
1260
1261   void dump(Formatter *f) const;
1262 };
1263
1264 class RGWEnv;
1265
1266 /* Namespaced forward declarations. */
1267 namespace rgw {
1268   namespace auth {
1269     namespace s3 {
1270       class AWSBrowserUploadAbstractor;
1271       class STSEngine;
1272     }
1273     class Completer;
1274   }
1275   namespace io {
1276     class BasicClient;
1277   }
1278 }
1279
1280 using meta_map_t = boost::container::flat_map <std::string, std::string>;
1281
1282 struct req_info {
1283   const RGWEnv *env;
1284   RGWHTTPArgs args;
1285   meta_map_t x_meta_map;
1286
1287   string host;
1288   const char *method;
1289   string script_uri;
1290   string request_uri;
1291   string request_uri_aws4;
1292   string effective_uri;
1293   string request_params;
1294   string domain;
1295   string storage_class;
1296
1297   req_info(CephContext *cct, const RGWEnv *env);
1298   void rebuild_from(req_info& src);
1299   void init_meta_info(bool *found_bad_meta);
1300 };
1301
1302 typedef cls_rgw_obj_key rgw_obj_index_key;
1303
1304 struct rgw_obj_key {
1305   string name;
1306   string instance;
1307   string ns;
1308
1309   rgw_obj_key() {}
1310   // cppcheck-suppress noExplicitConstructor
1311   rgw_obj_key(const string& n) : name(n) {}
1312   rgw_obj_key(const string& n, const string& i) : name(n), instance(i) {}
1313   rgw_obj_key(const string& n, const string& i, const string& _ns) : name(n), instance(i), ns(_ns) {}
1314
1315   rgw_obj_key(const rgw_obj_index_key& k) {
1316     parse_index_key(k.name, &name, &ns);
1317     instance = k.instance;
1318   }
1319
1320   static void parse_index_key(const string& key, string *name, string *ns) {
1321     if (key[0] != '_') {
1322       *name = key;
1323       ns->clear();
1324       return;
1325     }
1326     if (key[1] == '_') {
1327       *name = key.substr(1);
1328       ns->clear();
1329       return;
1330     }
1331     ssize_t pos = key.find('_', 1);
1332     if (pos < 0) {
1333       /* shouldn't happen, just use key */
1334       *name = key;
1335       ns->clear();
1336       return;
1337     }
1338
1339     *name = key.substr(pos + 1);
1340     *ns = key.substr(1, pos -1);
1341   }
1342
1343   void set(const string& n) {
1344     name = n;
1345     instance.clear();
1346     ns.clear();
1347   }
1348
1349   void set(const string& n, const string& i) {
1350     name = n;
1351     instance = i;
1352     ns.clear();
1353   }
1354
1355   void set(const string& n, const string& i, const string& _ns) {
1356     name = n;
1357     instance = i;
1358     ns = _ns;
1359   }
1360
1361   bool set(const rgw_obj_index_key& index_key) {
1362     if (!parse_raw_oid(index_key.name, this)) {
1363       return false;
1364     }
1365     instance = index_key.instance;
1366     return true;
1367   }
1368
1369   void set_instance(const string& i) {
1370     instance = i;
1371   }
1372
1373   const string& get_instance() const {
1374     return instance;
1375   }
1376
1377   void set_ns(const std::string& _ns) {
1378     ns = _ns;
1379   }
1380
1381   const std::string& get_ns() const {
1382     return ns;
1383   }
1384
1385   string get_index_key_name() const {
1386     if (ns.empty()) {
1387       if (name.size() < 1 || name[0] != '_') {
1388         return name;
1389       }
1390       return string("_") + name;
1391     };
1392
1393     char buf[ns.size() + 16];
1394     snprintf(buf, sizeof(buf), "_%s_", ns.c_str());
1395     return string(buf) + name;
1396   };
1397
1398   void get_index_key(rgw_obj_index_key *key) const {
1399     key->name = get_index_key_name();
1400     key->instance = instance;
1401   }
1402
1403   string get_loc() const {
1404     /*
1405      * For backward compatibility. Older versions used to have object locator on all objects,
1406      * however, the name was the effective object locator. This had the same effect as not
1407      * having object locator at all for most objects but the ones that started with underscore as
1408      * these were escaped.
1409      */
1410     if (name[0] == '_' && ns.empty()) {
1411       return name;
1412     }
1413
1414     return string();
1415   }
1416
1417   bool empty() const {
1418     return name.empty();
1419   }
1420
1421   bool have_null_instance() const {
1422     return instance == "null";
1423   }
1424
1425   bool have_instance() const {
1426     return !instance.empty();
1427   }
1428
1429   bool need_to_encode_instance() const {
1430     return have_instance() && !have_null_instance();
1431   }
1432
1433   string get_oid() const {
1434     if (ns.empty() && !need_to_encode_instance()) {
1435       if (name.size() < 1 || name[0] != '_') {
1436         return name;
1437       }
1438       return string("_") + name;
1439     }
1440
1441     string oid = "_";
1442     oid.append(ns);
1443     if (need_to_encode_instance()) {
1444       oid.append(string(":") + instance);
1445     }
1446     oid.append("_");
1447     oid.append(name);
1448     return oid;
1449   }
1450
1451   bool operator==(const rgw_obj_key& k) const {
1452     return (name.compare(k.name) == 0) &&
1453            (instance.compare(k.instance) == 0);
1454   }
1455
1456   bool operator<(const rgw_obj_key& k) const {
1457     int r = name.compare(k.name);
1458     if (r == 0) {
1459       r = instance.compare(k.instance);
1460     }
1461     return (r < 0);
1462   }
1463
1464   bool operator<=(const rgw_obj_key& k) const {
1465     return !(k < *this);
1466   }
1467
1468   static void parse_ns_field(string& ns, string& instance) {
1469     int pos = ns.find(':');
1470     if (pos >= 0) {
1471       instance = ns.substr(pos + 1);
1472       ns = ns.substr(0, pos);
1473     } else {
1474       instance.clear();
1475     }
1476   }
1477
1478   // takes an oid and parses out the namespace (ns), name, and
1479   // instance
1480   static bool parse_raw_oid(const string& oid, rgw_obj_key *key) {
1481     key->instance.clear();
1482     key->ns.clear();
1483     if (oid[0] != '_') {
1484       key->name = oid;
1485       return true;
1486     }
1487
1488     if (oid.size() >= 2 && oid[1] == '_') {
1489       key->name = oid.substr(1);
1490       return true;
1491     }
1492
1493     if (oid.size() < 3) // for namespace, min size would be 3: _x_
1494       return false;
1495
1496     size_t pos = oid.find('_', 2); // oid must match ^_[^_].+$
1497     if (pos == string::npos)
1498       return false;
1499
1500     key->ns = oid.substr(1, pos - 1);
1501     parse_ns_field(key->ns, key->instance);
1502
1503     key->name = oid.substr(pos + 1);
1504     return true;
1505   }
1506
1507   /**
1508    * Translate a namespace-mangled object name to the user-facing name
1509    * existing in the given namespace.
1510    *
1511    * If the object is part of the given namespace, it returns true
1512    * and cuts down the name to the unmangled version. If it is not
1513    * part of the given namespace, it returns false.
1514    */
1515   static bool oid_to_key_in_ns(const string& oid, rgw_obj_key *key, const string& ns) {
1516     bool ret = parse_raw_oid(oid, key);
1517     if (!ret) {
1518       return ret;
1519     }
1520
1521     return (ns == key->ns);
1522   }
1523
1524   /**
1525    * Given a mangled object name and an empty namespace string, this
1526    * function extracts the namespace into the string and sets the object
1527    * name to be the unmangled version.
1528    *
1529    * It returns true after successfully doing so, or
1530    * false if it fails.
1531    */
1532   static bool strip_namespace_from_name(string& name, string& ns, string& instance) {
1533     ns.clear();
1534     instance.clear();
1535     if (name[0] != '_') {
1536       return true;
1537     }
1538
1539     size_t pos = name.find('_', 1);
1540     if (pos == string::npos) {
1541       return false;
1542     }
1543
1544     if (name[1] == '_') {
1545       name = name.substr(1);
1546       return true;
1547     }
1548
1549     size_t period_pos = name.find('.');
1550     if (period_pos < pos) {
1551       return false;
1552     }
1553
1554     ns = name.substr(1, pos-1);
1555     name = name.substr(pos+1, string::npos);
1556
1557     parse_ns_field(ns, instance);
1558     return true;
1559   }
1560
1561   void encode(bufferlist& bl) const {
1562     ENCODE_START(2, 1, bl);
1563     encode(name, bl);
1564     encode(instance, bl);
1565     encode(ns, bl);
1566     ENCODE_FINISH(bl);
1567   }
1568   void decode(bufferlist::const_iterator& bl) {
1569     DECODE_START(2, bl);
1570     decode(name, bl);
1571     decode(instance, bl);
1572     if (struct_v >= 2) {
1573       decode(ns, bl);
1574     }
1575     DECODE_FINISH(bl);
1576   }
1577   void dump(Formatter *f) const;
1578   void decode_json(JSONObj *obj);
1579
1580   string to_str() const {
1581     if (instance.empty()) {
1582       return name;
1583     }
1584     char buf[name.size() + instance.size() + 16];
1585     snprintf(buf, sizeof(buf), "%s[%s]", name.c_str(), instance.c_str());
1586     return buf;
1587   }
1588 };
1589 WRITE_CLASS_ENCODER(rgw_obj_key)
1590
1591 inline ostream& operator<<(ostream& out, const rgw_obj_key &o) {
1592   return out << o.to_str();
1593 }
1594
1595 inline ostream& operator<<(ostream& out, const rgw_obj_index_key &o) {
1596   if (o.instance.empty()) {
1597     return out << o.name;
1598   } else {
1599     return out << o.name << "[" << o.instance << "]";
1600   }
1601 }
1602
1603 struct req_init_state {
1604   /* Keeps [[tenant]:]bucket until we parse the token. */
1605   string url_bucket;
1606   string src_bucket;
1607 };
1608
1609 #include "rgw_auth.h"
1610
1611 class RGWObjectCtx;
1612 class RGWSysObjectCtx;
1613
1614 /** Store all the state necessary to complete and respond to an HTTP request*/
1615 struct req_state : DoutPrefixProvider {
1616   CephContext *cct;
1617   rgw::io::BasicClient *cio{nullptr};
1618   http_op op{OP_UNKNOWN};
1619   RGWOpType op_type{};
1620   bool content_started{false};
1621   int format{0};
1622   ceph::Formatter *formatter{nullptr};
1623   string decoded_uri;
1624   string relative_uri;
1625   const char *length{nullptr};
1626   int64_t content_length{0};
1627   map<string, string> generic_attrs;
1628   rgw_err err;
1629   bool expect_cont{false};
1630   uint64_t obj_size{0};
1631   bool enable_ops_log;
1632   bool enable_usage_log;
1633   uint8_t defer_to_bucket_acls;
1634   uint32_t perm_mask{0};
1635
1636   /* Set once when url_bucket is parsed and not violated thereafter. */
1637   string account_name;
1638
1639   string bucket_tenant;
1640   string bucket_name;
1641
1642   rgw_bucket bucket;
1643   rgw_obj_key object;
1644   string src_tenant_name;
1645   string src_bucket_name;
1646   rgw_obj_key src_object;
1647   ACLOwner bucket_owner;
1648   ACLOwner owner;
1649
1650   string zonegroup_name;
1651   string zonegroup_endpoint;
1652   string bucket_instance_id;
1653   int bucket_instance_shard_id{-1};
1654   string redirect_zone_endpoint;
1655
1656   string redirect;
1657
1658   RGWBucketInfo bucket_info;
1659   obj_version bucket_ep_objv;
1660   real_time bucket_mtime;
1661   std::map<std::string, ceph::bufferlist> bucket_attrs;
1662   bool bucket_exists{false};
1663   rgw_placement_rule dest_placement;
1664
1665   bool has_bad_meta{false};
1666
1667   rgw::sal::RGWUser *user;
1668
1669   struct {
1670     /* TODO(rzarzynski): switch out to the static_ptr for both members. */
1671
1672     /* Object having the knowledge about an authenticated identity and allowing
1673      * to apply it during the authorization phase (verify_permission() methods
1674      * of a given RGWOp). Thus, it bounds authentication and authorization steps
1675      * through a well-defined interface. For more details, see rgw_auth.h. */
1676     std::unique_ptr<rgw::auth::Identity> identity;
1677
1678     std::shared_ptr<rgw::auth::Completer> completer;
1679
1680     /* A container for credentials of the S3's browser upload. It's necessary
1681      * because: 1) the ::authenticate() method of auth engines and strategies
1682      * take req_state only; 2) auth strategies live much longer than RGWOps -
1683      * there is no way to pass additional data dependencies through ctors. */
1684     class {
1685       /* Writer. */
1686       friend class RGWPostObj_ObjStore_S3;
1687       /* Reader. */
1688       friend class rgw::auth::s3::AWSBrowserUploadAbstractor;
1689       friend class rgw::auth::s3::STSEngine;
1690
1691       std::string access_key;
1692       std::string signature;
1693       std::string x_amz_algorithm;
1694       std::string x_amz_credential;
1695       std::string x_amz_date;
1696       std::string x_amz_security_token;
1697       ceph::bufferlist encoded_policy;
1698     } s3_postobj_creds;
1699   } auth;
1700
1701   std::unique_ptr<RGWAccessControlPolicy> user_acl;
1702   std::unique_ptr<RGWAccessControlPolicy> bucket_acl;
1703   std::unique_ptr<RGWAccessControlPolicy> object_acl;
1704
1705   rgw::IAM::Environment env;
1706   boost::optional<rgw::IAM::Policy> iam_policy;
1707   boost::optional<PublicAccessBlockConfiguration> bucket_access_conf;
1708   vector<rgw::IAM::Policy> iam_user_policies;
1709
1710   /* Is the request made by an user marked as a system one?
1711    * Being system user means we also have the admin status. */
1712   bool system_request{false};
1713
1714   string canned_acl;
1715   bool has_acl_header{false};
1716   bool local_source{false}; /* source is local */
1717
1718   int prot_flags{0};
1719
1720   /* Content-Disposition override for TempURL of Swift API. */
1721   struct {
1722     string override;
1723     string fallback;
1724   } content_disp;
1725
1726   string host_id;
1727
1728   req_info info;
1729   req_init_state init_state;
1730
1731   using Clock = ceph::coarse_real_clock;
1732   Clock::time_point time;
1733
1734   Clock::duration time_elapsed() const { return Clock::now() - time; }
1735
1736   RGWObjectCtx *obj_ctx{nullptr};
1737   RGWSysObjectCtx *sysobj_ctx{nullptr};
1738   string dialect;
1739   string req_id;
1740   string trans_id;
1741   uint64_t id;
1742
1743   RGWObjTags tagset;
1744
1745   bool mfa_verified{false};
1746
1747   /// optional coroutine context
1748   optional_yield yield{null_yield};
1749
1750   req_state(CephContext* _cct, RGWEnv* e, rgw::sal::RGWUser* u, uint64_t id);
1751   ~req_state();
1752
1753   bool is_err() const { return err.is_err(); }
1754
1755   // implements DoutPrefixProvider
1756   std::ostream& gen_prefix(std::ostream& out) const override;
1757   CephContext* get_cct() const override { return cct; }
1758   unsigned get_subsys() const override { return ceph_subsys_rgw; }
1759 };
1760
1761 void set_req_state_err(struct req_state*, int);
1762 void set_req_state_err(struct req_state*, int, const string&);
1763 void set_req_state_err(struct rgw_err&, int, const int);
1764 void dump(struct req_state*);
1765
1766 /** Store basic data on bucket */
1767 struct RGWBucketEnt {
1768   rgw_bucket bucket;
1769   size_t size;
1770   size_t size_rounded;
1771   ceph::real_time creation_time;
1772   uint64_t count;
1773
1774   /* The placement_rule is necessary to calculate per-storage-policy statics
1775    * of the Swift API. Although the info available in RGWBucketInfo, we need
1776    * to duplicate it here to not affect the performance of buckets listing. */
1777   rgw_placement_rule placement_rule;
1778
1779   RGWBucketEnt()
1780     : size(0),
1781       size_rounded(0),
1782       count(0) {
1783   }
1784   RGWBucketEnt(const RGWBucketEnt&) = default;
1785   RGWBucketEnt(RGWBucketEnt&&) = default;
1786   explicit RGWBucketEnt(const rgw_user& u, cls_user_bucket_entry&& e)
1787     : bucket(u, std::move(e.bucket)),
1788       size(e.size),
1789       size_rounded(e.size_rounded),
1790       creation_time(e.creation_time),
1791       count(e.count) {
1792   }
1793
1794   RGWBucketEnt& operator=(const RGWBucketEnt&) = default;
1795
1796   void convert(cls_user_bucket_entry *b) const {
1797     bucket.convert(&b->bucket);
1798     b->size = size;
1799     b->size_rounded = size_rounded;
1800     b->creation_time = creation_time;
1801     b->count = count;
1802   }
1803
1804   void encode(bufferlist& bl) const {
1805     ENCODE_START(7, 5, bl);
1806     uint64_t s = size;
1807     __u32 mt = ceph::real_clock::to_time_t(creation_time);
1808     string empty_str;  // originally had the bucket name here, but we encode bucket later
1809     encode(empty_str, bl);
1810     encode(s, bl);
1811     encode(mt, bl);
1812     encode(count, bl);
1813     encode(bucket, bl);
1814     s = size_rounded;
1815     encode(s, bl);
1816     encode(creation_time, bl);
1817     encode(placement_rule, bl);
1818     ENCODE_FINISH(bl);
1819   }
1820   void decode(bufferlist::const_iterator& bl) {
1821     DECODE_START_LEGACY_COMPAT_LEN(7, 5, 5, bl);
1822     __u32 mt;
1823     uint64_t s;
1824     string empty_str;  // backward compatibility
1825     decode(empty_str, bl);
1826     decode(s, bl);
1827     decode(mt, bl);
1828     size = s;
1829     if (struct_v < 6) {
1830       creation_time = ceph::real_clock::from_time_t(mt);
1831     }
1832     if (struct_v >= 2)
1833       decode(count, bl);
1834     if (struct_v >= 3)
1835       decode(bucket, bl);
1836     if (struct_v >= 4)
1837       decode(s, bl);
1838     size_rounded = s;
1839     if (struct_v >= 6)
1840       decode(creation_time, bl);
1841     if (struct_v >= 7)
1842       decode(placement_rule, bl);
1843     DECODE_FINISH(bl);
1844   }
1845   void dump(Formatter *f) const;
1846   static void generate_test_instances(list<RGWBucketEnt*>& o);
1847 };
1848 WRITE_CLASS_ENCODER(RGWBucketEnt)
1849
1850 struct rgw_obj {
1851   rgw_bucket bucket;
1852   rgw_obj_key key;
1853
1854   bool in_extra_data{false}; /* in-memory only member, does not serialize */
1855
1856   // Represents the hash index source for this object once it is set (non-empty)
1857   std::string index_hash_source;
1858
1859   rgw_obj() {}
1860   rgw_obj(const rgw_bucket& b, const std::string& name) : bucket(b), key(name) {}
1861   rgw_obj(const rgw_bucket& b, const rgw_obj_key& k) : bucket(b), key(k) {}
1862   rgw_obj(const rgw_bucket& b, const rgw_obj_index_key& k) : bucket(b), key(k) {}
1863
1864   void init(const rgw_bucket& b, const std::string& name) {
1865     bucket = b;
1866     key.set(name);
1867   }
1868   void init(const rgw_bucket& b, const std::string& name, const string& i, const string& n) {
1869     bucket = b;
1870     key.set(name, i, n);
1871   }
1872   void init_ns(const rgw_bucket& b, const std::string& name, const string& n) {
1873     bucket = b;
1874     key.name = name;
1875     key.instance.clear();
1876     key.ns = n;
1877   }
1878
1879   bool empty() const {
1880     return key.empty();
1881   }
1882
1883   void set_key(const rgw_obj_key& k) {
1884     key = k;
1885   }
1886
1887   string get_oid() const {
1888     return key.get_oid();
1889   }
1890
1891   const string& get_hash_object() const {
1892     return index_hash_source.empty() ? key.name : index_hash_source;
1893   }
1894
1895   void set_in_extra_data(bool val) {
1896     in_extra_data = val;
1897   }
1898
1899   bool is_in_extra_data() const {
1900     return in_extra_data;
1901   }
1902
1903   void encode(bufferlist& bl) const {
1904     ENCODE_START(6, 6, bl);
1905     encode(bucket, bl);
1906     encode(key.ns, bl);
1907     encode(key.name, bl);
1908     encode(key.instance, bl);
1909 //    encode(placement_id, bl);
1910     ENCODE_FINISH(bl);
1911   }
1912   void decode(bufferlist::const_iterator& bl) {
1913     DECODE_START_LEGACY_COMPAT_LEN(6, 3, 3, bl);
1914     if (struct_v < 6) {
1915       string s;
1916       decode(bucket.name, bl); /* bucket.name */
1917       decode(s, bl); /* loc */
1918       decode(key.ns, bl);
1919       decode(key.name, bl);
1920       if (struct_v >= 2)
1921         decode(bucket, bl);
1922       if (struct_v >= 4)
1923         decode(key.instance, bl);
1924       if (key.ns.empty() && key.instance.empty()) {
1925         if (key.name[0] == '_') {
1926           key.name = key.name.substr(1);
1927         }
1928       } else {
1929         if (struct_v >= 5) {
1930           decode(key.name, bl);
1931         } else {
1932           ssize_t pos = key.name.find('_', 1);
1933           if (pos < 0) {
1934             throw buffer::error();
1935           }
1936           key.name = key.name.substr(pos + 1);
1937         }
1938       }
1939     } else {
1940       decode(bucket, bl);
1941       decode(key.ns, bl);
1942       decode(key.name, bl);
1943       decode(key.instance, bl);
1944 //      decode(placement_id, bl);
1945     }
1946     DECODE_FINISH(bl);
1947   }
1948   void dump(Formatter *f) const;
1949   static void generate_test_instances(list<rgw_obj*>& o);
1950
1951   bool operator==(const rgw_obj& o) const {
1952     return (key == o.key) &&
1953            (bucket == o.bucket);
1954   }
1955   bool operator<(const rgw_obj& o) const {
1956     int r = key.name.compare(o.key.name);
1957     if (r == 0) {
1958       r = bucket.bucket_id.compare(o.bucket.bucket_id); /* not comparing bucket.name, if bucket_id is equal so will be bucket.name */
1959       if (r == 0) {
1960         r = key.ns.compare(o.key.ns);
1961         if (r == 0) {
1962           r = key.instance.compare(o.key.instance);
1963         }
1964       }
1965     }
1966
1967     return (r < 0);
1968   }
1969
1970   const rgw_pool& get_explicit_data_pool() {
1971     if (!in_extra_data || bucket.explicit_placement.data_extra_pool.empty()) {
1972       return bucket.explicit_placement.data_pool;
1973     }
1974     return bucket.explicit_placement.data_extra_pool;
1975   }
1976 };
1977 WRITE_CLASS_ENCODER(rgw_obj)
1978
1979 struct rgw_cache_entry_info {
1980   string cache_locator;
1981   uint64_t gen;
1982
1983   rgw_cache_entry_info() : gen(0) {}
1984 };
1985
1986 inline ostream& operator<<(ostream& out, const rgw_obj &o) {
1987   return out << o.bucket.name << ":" << o.get_oid();
1988 }
1989
1990 static inline void buf_to_hex(const unsigned char* const buf,
1991                               const size_t len,
1992                               char* const str)
1993 {
1994   str[0] = '\0';
1995   for (size_t i = 0; i < len; i++) {
1996     ::sprintf(&str[i*2], "%02x", static_cast<int>(buf[i]));
1997   }
1998 }
1999
2000 template<size_t N> static inline std::array<char, N * 2 + 1>
2001 buf_to_hex(const std::array<unsigned char, N>& buf)
2002 {
2003   static_assert(N > 0, "The input array must be at least one element long");
2004
2005   std::array<char, N * 2 + 1> hex_dest;
2006   buf_to_hex(buf.data(), N, hex_dest.data());
2007   return hex_dest;
2008 }
2009
2010 static inline int hexdigit(char c)
2011 {
2012   if (c >= '0' && c <= '9')
2013     return (c - '0');
2014   c = toupper(c);
2015   if (c >= 'A' && c <= 'F')
2016     return c - 'A' + 0xa;
2017   return -EINVAL;
2018 }
2019
2020 static inline int hex_to_buf(const char *hex, char *buf, int len)
2021 {
2022   int i = 0;
2023   const char *p = hex;
2024   while (*p) {
2025     if (i >= len)
2026       return -EINVAL;
2027     buf[i] = 0;
2028     int d = hexdigit(*p);
2029     if (d < 0)
2030       return d;
2031     buf[i] = d << 4;
2032     p++;
2033     if (!*p)
2034       return -EINVAL;
2035     d = hexdigit(*p);
2036     if (d < 0)
2037       return d;
2038     buf[i] += d;
2039     i++;
2040     p++;
2041   }
2042   return i;
2043 }
2044
2045 static inline int rgw_str_to_bool(const char *s, int def_val)
2046 {
2047   if (!s)
2048     return def_val;
2049
2050   return (strcasecmp(s, "true") == 0 ||
2051           strcasecmp(s, "on") == 0 ||
2052           strcasecmp(s, "yes") == 0 ||
2053           strcasecmp(s, "1") == 0);
2054 }
2055
2056 static inline void append_rand_alpha(CephContext *cct, const string& src, string& dest, int len)
2057 {
2058   dest = src;
2059   char buf[len + 1];
2060   gen_rand_alphanumeric(cct, buf, len);
2061   dest.append("_");
2062   dest.append(buf);
2063 }
2064
2065 static inline const char *rgw_obj_category_name(RGWObjCategory category)
2066 {
2067   switch (category) {
2068   case RGWObjCategory::None:
2069     return "rgw.none";
2070   case RGWObjCategory::Main:
2071     return "rgw.main";
2072   case RGWObjCategory::Shadow:
2073     return "rgw.shadow";
2074   case RGWObjCategory::MultiMeta:
2075     return "rgw.multimeta";
2076   }
2077
2078   return "unknown";
2079 }
2080
2081 static inline uint64_t rgw_rounded_kb(uint64_t bytes)
2082 {
2083   return (bytes + 1023) / 1024;
2084 }
2085
2086 static inline uint64_t rgw_rounded_objsize(uint64_t bytes)
2087 {
2088   return ((bytes + 4095) & ~4095);
2089 }
2090
2091 static inline uint64_t rgw_rounded_objsize_kb(uint64_t bytes)
2092 {
2093   return ((bytes + 4095) & ~4095) / 1024;
2094 }
2095
2096 /* implement combining step, S3 header canonicalization;  k is a
2097  * valid header and in lc form */
2098 void rgw_add_amz_meta_header(
2099   meta_map_t& x_meta_map,
2100   const std::string& k,
2101   const std::string& v);
2102
2103 extern string rgw_string_unquote(const string& s);
2104 extern void parse_csv_string(const string& ival, vector<string>& ovals);
2105 extern int parse_key_value(string& in_str, string& key, string& val);
2106 extern int parse_key_value(string& in_str, const char *delim, string& key, string& val);
2107
2108 extern boost::optional<std::pair<boost::string_view, boost::string_view>>
2109 parse_key_value(const boost::string_view& in_str,
2110                 const boost::string_view& delim);
2111 extern boost::optional<std::pair<boost::string_view, boost::string_view>>
2112 parse_key_value(const boost::string_view& in_str);
2113
2114
2115 /** time parsing */
2116 extern int parse_time(const char *time_str, real_time *time);
2117 extern bool parse_rfc2616(const char *s, struct tm *t);
2118 extern bool parse_iso8601(const char *s, struct tm *t, uint32_t *pns = NULL, bool extended_format = true);
2119 extern string rgw_trim_whitespace(const string& src);
2120 extern boost::string_view rgw_trim_whitespace(const boost::string_view& src);
2121 extern string rgw_trim_quotes(const string& val);
2122
2123 extern void rgw_to_iso8601(const real_time& t, char *dest, int buf_size);
2124 extern void rgw_to_iso8601(const real_time& t, string *dest);
2125 extern std::string rgw_to_asctime(const utime_t& t);
2126
2127 struct perm_state_base {
2128   CephContext *cct;
2129   const rgw::IAM::Environment& env;
2130   rgw::auth::Identity *identity;
2131   const RGWBucketInfo& bucket_info;
2132   int perm_mask;
2133   bool defer_to_bucket_acls;
2134   boost::optional<PublicAccessBlockConfiguration> bucket_access_conf;
2135
2136   perm_state_base(CephContext *_cct,
2137                   const rgw::IAM::Environment& _env,
2138                   rgw::auth::Identity *_identity,
2139                   const RGWBucketInfo& _bucket_info,
2140                   int _perm_mask,
2141                   bool _defer_to_bucket_acls,
2142                   boost::optional<PublicAccessBlockConfiguration> _bucket_acess_conf = boost::none) :
2143                                                 cct(_cct),
2144                                                 env(_env),
2145                                                 identity(_identity),
2146                                                 bucket_info(_bucket_info),
2147                                                 perm_mask(_perm_mask),
2148                                                 defer_to_bucket_acls(_defer_to_bucket_acls),
2149                                                 bucket_access_conf(_bucket_acess_conf)
2150   {}
2151
2152   virtual ~perm_state_base() {}
2153
2154   virtual const char *get_referer() const = 0;
2155   virtual std::optional<bool> get_request_payer() const = 0; /*
2156                                                               * empty state means that request_payer param was not passed in
2157                                                               */
2158
2159 };
2160
2161 struct perm_state : public perm_state_base {
2162   const char *referer;
2163   bool request_payer;
2164
2165   perm_state(CephContext *_cct,
2166              const rgw::IAM::Environment& _env,
2167              rgw::auth::Identity *_identity,
2168              const RGWBucketInfo& _bucket_info,
2169              int _perm_mask,
2170              bool _defer_to_bucket_acls,
2171              const char *_referer,
2172              bool _request_payer) : perm_state_base(_cct,
2173                                                     _env,
2174                                                     _identity,
2175                                                     _bucket_info,
2176                                                     _perm_mask,
2177                                                     _defer_to_bucket_acls),
2178                                     referer(_referer),
2179                                     request_payer(_request_payer) {}
2180
2181   const char *get_referer() const override {
2182     return referer;
2183   }
2184
2185   std::optional<bool> get_request_payer() const override {
2186     return request_payer;
2187   }
2188 };
2189
2190 /** Check if the req_state's user has the necessary permissions
2191  * to do the requested action  */
2192 bool verify_bucket_permission_no_policy(
2193   const DoutPrefixProvider* dpp,
2194   struct perm_state_base * const s,
2195   RGWAccessControlPolicy * const user_acl,
2196   RGWAccessControlPolicy * const bucket_acl,
2197   const int perm);
2198
2199 bool verify_user_permission_no_policy(const DoutPrefixProvider* dpp,
2200                                       struct perm_state_base * const s,
2201                                       RGWAccessControlPolicy * const user_acl,
2202                                       const int perm);
2203
2204 bool verify_object_permission_no_policy(const DoutPrefixProvider* dpp,
2205                                         struct perm_state_base * const s,
2206                                         RGWAccessControlPolicy * const user_acl,
2207                                         RGWAccessControlPolicy * const bucket_acl,
2208                                         RGWAccessControlPolicy * const object_acl,
2209                                         const int perm);
2210
2211 /** Check if the req_state's user has the necessary permissions
2212  * to do the requested action */
2213 rgw::IAM::Effect eval_user_policies(const vector<rgw::IAM::Policy>& user_policies,
2214                           const rgw::IAM::Environment& env,
2215                           boost::optional<const rgw::auth::Identity&> id,
2216                           const uint64_t op,
2217                           const rgw::ARN& arn);
2218 bool verify_user_permission(const DoutPrefixProvider* dpp,
2219                             struct req_state * const s,
2220                             RGWAccessControlPolicy * const user_acl,
2221                             const vector<rgw::IAM::Policy>& user_policies,
2222                             const rgw::ARN& res,
2223                             const uint64_t op);
2224 bool verify_user_permission_no_policy(const DoutPrefixProvider* dpp,
2225                                       struct req_state * const s,
2226                                       RGWAccessControlPolicy * const user_acl,
2227                                       const int perm);
2228 bool verify_user_permission(const DoutPrefixProvider* dpp,
2229                             struct req_state * const s,
2230                             const rgw::ARN& res,
2231                             const uint64_t op);
2232 bool verify_user_permission_no_policy(const DoutPrefixProvider* dpp,
2233                                       struct req_state * const s,
2234                                       int perm);
2235 bool verify_bucket_permission(
2236   const DoutPrefixProvider* dpp,
2237   struct req_state * const s,
2238   const rgw_bucket& bucket,
2239   RGWAccessControlPolicy * const user_acl,
2240   RGWAccessControlPolicy * const bucket_acl,
2241   const boost::optional<rgw::IAM::Policy>& bucket_policy,
2242   const vector<rgw::IAM::Policy>& user_policies,
2243   const uint64_t op);
2244 bool verify_bucket_permission(const DoutPrefixProvider* dpp, struct req_state * const s, const uint64_t op);
2245 bool verify_bucket_permission_no_policy(
2246   const DoutPrefixProvider* dpp,
2247   struct req_state * const s,
2248   RGWAccessControlPolicy * const user_acl,
2249   RGWAccessControlPolicy * const bucket_acl,
2250   const int perm);
2251 bool verify_bucket_permission_no_policy(const DoutPrefixProvider* dpp,
2252                                         struct req_state * const s,
2253                                         const int perm);
2254 int verify_bucket_owner_or_policy(struct req_state* const s,
2255                                   const uint64_t op);
2256 extern bool verify_object_permission(
2257   const DoutPrefixProvider* dpp,
2258   struct req_state * const s,
2259   const rgw_obj& obj,
2260   RGWAccessControlPolicy * const user_acl,
2261   RGWAccessControlPolicy * const bucket_acl,
2262   RGWAccessControlPolicy * const object_acl,
2263   const boost::optional<rgw::IAM::Policy>& bucket_policy,
2264   const vector<rgw::IAM::Policy>& user_policies,
2265   const uint64_t op);
2266 extern bool verify_object_permission(const DoutPrefixProvider* dpp, struct req_state *s, uint64_t op);
2267 extern bool verify_object_permission_no_policy(
2268   const DoutPrefixProvider* dpp,
2269   struct req_state * const s,
2270   RGWAccessControlPolicy * const user_acl,
2271   RGWAccessControlPolicy * const bucket_acl,
2272   RGWAccessControlPolicy * const object_acl,
2273   int perm);
2274 extern bool verify_object_permission_no_policy(const DoutPrefixProvider* dpp, struct req_state *s,
2275                                                int perm);
2276 /** Convert an input URL into a sane object name
2277  * by converting %-escaped strings into characters, etc*/
2278 extern void rgw_uri_escape_char(char c, string& dst);
2279 extern std::string url_decode(const boost::string_view& src_str,
2280                               bool in_query = false);
2281 extern void url_encode(const std::string& src, string& dst,
2282                        bool encode_slash = true);
2283 extern std::string url_encode(const std::string& src, bool encode_slash = true);
2284 extern std::string url_remove_prefix(const std::string& url); // Removes hhtp, https and www from url
2285 /* destination should be CEPH_CRYPTO_HMACSHA1_DIGESTSIZE bytes long */
2286 extern void calc_hmac_sha1(const char *key, int key_len,
2287                           const char *msg, int msg_len, char *dest);
2288
2289 static inline sha1_digest_t
2290 calc_hmac_sha1(const boost::string_view& key, const boost::string_view& msg) {
2291   sha1_digest_t dest;
2292   calc_hmac_sha1(key.data(), key.size(), msg.data(), msg.size(),
2293                  reinterpret_cast<char*>(dest.v));
2294   return dest;
2295 }
2296
2297 /* destination should be CEPH_CRYPTO_HMACSHA256_DIGESTSIZE bytes long */
2298 extern void calc_hmac_sha256(const char *key, int key_len,
2299                              const char *msg, int msg_len,
2300                              char *dest);
2301
2302 static inline sha256_digest_t
2303 calc_hmac_sha256(const char *key, const int key_len,
2304                  const char *msg, const int msg_len) {
2305   sha256_digest_t dest;
2306   calc_hmac_sha256(key, key_len, msg, msg_len,
2307                    reinterpret_cast<char*>(dest.v));
2308   return dest;
2309 }
2310
2311 static inline sha256_digest_t
2312 calc_hmac_sha256(const boost::string_view& key, const boost::string_view& msg) {
2313   sha256_digest_t dest;
2314   calc_hmac_sha256(key.data(), key.size(),
2315                    msg.data(), msg.size(),
2316                    reinterpret_cast<char*>(dest.v));
2317   return dest;
2318 }
2319
2320 static inline sha256_digest_t
2321 calc_hmac_sha256(const sha256_digest_t &key,
2322                  const boost::string_view& msg) {
2323   sha256_digest_t dest;
2324   calc_hmac_sha256(reinterpret_cast<const char*>(key.v), sha256_digest_t::SIZE,
2325                    msg.data(), msg.size(),
2326                    reinterpret_cast<char*>(dest.v));
2327   return dest;
2328 }
2329
2330 static inline sha256_digest_t
2331 calc_hmac_sha256(const std::vector<unsigned char>& key,
2332                  const boost::string_view& msg) {
2333   sha256_digest_t dest;
2334   calc_hmac_sha256(reinterpret_cast<const char*>(key.data()), key.size(),
2335                    msg.data(), msg.size(),
2336                    reinterpret_cast<char*>(dest.v));
2337   return dest;
2338 }
2339
2340 template<size_t KeyLenN>
2341 static inline sha256_digest_t
2342 calc_hmac_sha256(const std::array<unsigned char, KeyLenN>& key,
2343                  const boost::string_view& msg) {
2344   sha256_digest_t dest;
2345   calc_hmac_sha256(reinterpret_cast<const char*>(key.data()), key.size(),
2346                    msg.data(), msg.size(),
2347                    reinterpret_cast<char*>(dest.v));
2348   return dest;
2349 }
2350
2351 extern sha256_digest_t calc_hash_sha256(const boost::string_view& msg);
2352
2353 extern ceph::crypto::SHA256* calc_hash_sha256_open_stream();
2354 extern void calc_hash_sha256_update_stream(ceph::crypto::SHA256* hash,
2355                                            const char* msg,
2356                                            int len);
2357 extern std::string calc_hash_sha256_close_stream(ceph::crypto::SHA256** phash);
2358 extern std::string calc_hash_sha256_restart_stream(ceph::crypto::SHA256** phash);
2359
2360 extern int rgw_parse_op_type_list(const string& str, uint32_t *perm);
2361
2362 static constexpr uint32_t MATCH_POLICY_ACTION = 0x01;
2363 static constexpr uint32_t MATCH_POLICY_RESOURCE = 0x02;
2364 static constexpr uint32_t MATCH_POLICY_ARN = 0x04;
2365 static constexpr uint32_t MATCH_POLICY_STRING = 0x08;
2366
2367 extern bool match_policy(boost::string_view pattern, boost::string_view input,
2368                          uint32_t flag);
2369
2370 extern string camelcase_dash_http_attr(const string& orig);
2371 extern string lowercase_dash_http_attr(const string& orig);
2372
2373 void rgw_setup_saved_curl_handles();
2374 void rgw_release_all_curl_handles();
2375
2376 static inline void rgw_escape_str(const string& s, char esc_char,
2377                                   char special_char, string *dest)
2378 {
2379   const char *src = s.c_str();
2380   char dest_buf[s.size() * 2 + 1];
2381   char *destp = dest_buf;
2382
2383   for (size_t i = 0; i < s.size(); i++) {
2384     char c = src[i];
2385     if (c == esc_char || c == special_char) {
2386       *destp++ = esc_char;
2387     }
2388     *destp++ = c;
2389   }
2390   *destp++ = '\0';
2391   *dest = dest_buf;
2392 }
2393
2394 static inline ssize_t rgw_unescape_str(const string& s, ssize_t ofs,
2395                                        char esc_char, char special_char,
2396                                        string *dest)
2397 {
2398   const char *src = s.c_str();
2399   char dest_buf[s.size() + 1];
2400   char *destp = dest_buf;
2401   bool esc = false;
2402
2403   dest_buf[0] = '\0';
2404
2405   for (size_t i = ofs; i < s.size(); i++) {
2406     char c = src[i];
2407     if (!esc && c == esc_char) {
2408       esc = true;
2409       continue;
2410     }
2411     if (!esc && c == special_char) {
2412       *destp = '\0';
2413       *dest = dest_buf;
2414       return (ssize_t)i + 1;
2415     }
2416     *destp++ = c;
2417     esc = false;
2418   }
2419   *destp = '\0';
2420   *dest = dest_buf;
2421   return string::npos;
2422 }
2423
2424 static inline string rgw_bl_str(ceph::buffer::list& raw)
2425 {
2426   size_t len = raw.length();
2427   string s(raw.c_str(), len);
2428   while (len && !s[len - 1]) {
2429     --len;
2430     s.resize(len);
2431   }
2432   return s;
2433 }
2434
2435 template <typename T>
2436 int decode_bl(bufferlist& bl, T& t)
2437 {
2438   auto iter = bl.cbegin();
2439   try {
2440     decode(t, iter);
2441   } catch (buffer::error& err) {
2442     return -EIO;
2443   }
2444   return 0;
2445 }
2446
2447 #endif