1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab ft=cpp
5 * Ceph - scalable distributed file system
7 * Copyright (C) 2004-2009 Sage Weil <sage@newdream.net>
8 * Copyright (C) 2015 Yehuda Sadeh <yehuda@redhat.com>
10 * This is free software; you can redistribute it and/or
11 * modify it under the terms of the GNU Lesser General Public
12 * License version 2.1, as published by the Free Software
13 * Foundation. See file COPYING.
20 #include <string_view>
22 #include <unordered_map>
24 #include <fmt/format.h>
26 #include "common/ceph_crypto.h"
27 #include "common/random_string.h"
29 #include "rgw_bucket_layout.h"
31 #include "rgw_basic_types.h"
32 #include "rgw_iam_policy.h"
33 #include "rgw_quota_types.h"
34 #include "rgw_string.h"
35 #include "common/async/yield_context.h"
36 #include "rgw_website.h"
37 #include "rgw_object_lock.h"
39 #include "rgw_op_type.h"
40 #include "rgw_sync_policy.h"
41 #include "cls/version/cls_version_types.h"
42 #include "cls/user/cls_user_types.h"
43 #include "cls/rgw/cls_rgw_types.h"
44 #include "include/rados/librados.hpp"
45 #include "rgw_public_access.h"
46 #include "common/tracer.h"
47 #include "rgw_sal_fwd.h"
54 using Attrs
= std::map
<std::string
, ceph::buffer::list
>;
63 using ceph::crypto::MD5
;
65 #define RGW_ATTR_PREFIX "user.rgw."
67 #define RGW_HTTP_RGWX_ATTR_PREFIX "RGWX_ATTR_"
68 #define RGW_HTTP_RGWX_ATTR_PREFIX_OUT "Rgwx-Attr-"
70 #define RGW_AMZ_PREFIX "x-amz-"
71 #define RGW_AMZ_META_PREFIX RGW_AMZ_PREFIX "meta-"
72 #define RGW_AMZ_WEBSITE_REDIRECT_LOCATION RGW_AMZ_PREFIX "website-redirect-location"
73 #define RGW_AMZ_TAG_COUNT RGW_AMZ_PREFIX "tagging-count"
75 #define RGW_SYS_PARAM_PREFIX "rgwx-"
77 #define RGW_ATTR_ACL RGW_ATTR_PREFIX "acl"
78 #define RGW_ATTR_RATELIMIT RGW_ATTR_PREFIX "ratelimit"
79 #define RGW_ATTR_LC RGW_ATTR_PREFIX "lc"
80 #define RGW_ATTR_CORS RGW_ATTR_PREFIX "cors"
81 #define RGW_ATTR_ETAG RGW_ATTR_PREFIX "etag"
82 #define RGW_ATTR_BUCKETS RGW_ATTR_PREFIX "buckets"
83 #define RGW_ATTR_META_PREFIX RGW_ATTR_PREFIX RGW_AMZ_META_PREFIX
84 #define RGW_ATTR_CONTENT_TYPE RGW_ATTR_PREFIX "content_type"
85 #define RGW_ATTR_CACHE_CONTROL RGW_ATTR_PREFIX "cache_control"
86 #define RGW_ATTR_CONTENT_DISP RGW_ATTR_PREFIX "content_disposition"
87 #define RGW_ATTR_CONTENT_ENC RGW_ATTR_PREFIX "content_encoding"
88 #define RGW_ATTR_CONTENT_LANG RGW_ATTR_PREFIX "content_language"
89 #define RGW_ATTR_EXPIRES RGW_ATTR_PREFIX "expires"
90 #define RGW_ATTR_DELETE_AT RGW_ATTR_PREFIX "delete_at"
91 #define RGW_ATTR_ID_TAG RGW_ATTR_PREFIX "idtag"
92 #define RGW_ATTR_TAIL_TAG RGW_ATTR_PREFIX "tail_tag"
93 #define RGW_ATTR_SHADOW_OBJ RGW_ATTR_PREFIX "shadow_name"
94 #define RGW_ATTR_MANIFEST RGW_ATTR_PREFIX "manifest"
95 #define RGW_ATTR_USER_MANIFEST RGW_ATTR_PREFIX "user_manifest"
96 #define RGW_ATTR_AMZ_WEBSITE_REDIRECT_LOCATION RGW_ATTR_PREFIX RGW_AMZ_WEBSITE_REDIRECT_LOCATION
97 #define RGW_ATTR_SLO_MANIFEST RGW_ATTR_PREFIX "slo_manifest"
98 /* Information whether an object is SLO or not must be exposed to
99 * user through custom HTTP header named X-Static-Large-Object. */
100 #define RGW_ATTR_SLO_UINDICATOR RGW_ATTR_META_PREFIX "static-large-object"
101 #define RGW_ATTR_X_ROBOTS_TAG RGW_ATTR_PREFIX "x-robots-tag"
102 #define RGW_ATTR_STORAGE_CLASS RGW_ATTR_PREFIX "storage_class"
105 #define RGW_ATTR_OBJECT_LOCK RGW_ATTR_PREFIX "object-lock"
106 #define RGW_ATTR_OBJECT_RETENTION RGW_ATTR_PREFIX "object-retention"
107 #define RGW_ATTR_OBJECT_LEGAL_HOLD RGW_ATTR_PREFIX "object-legal-hold"
110 #define RGW_ATTR_PG_VER RGW_ATTR_PREFIX "pg_ver"
111 #define RGW_ATTR_SOURCE_ZONE RGW_ATTR_PREFIX "source_zone"
112 #define RGW_ATTR_TAGS RGW_ATTR_PREFIX RGW_AMZ_PREFIX "tagging"
114 #define RGW_ATTR_TEMPURL_KEY1 RGW_ATTR_META_PREFIX "temp-url-key"
115 #define RGW_ATTR_TEMPURL_KEY2 RGW_ATTR_META_PREFIX "temp-url-key-2"
117 /* Account/container quota of the Swift API. */
118 #define RGW_ATTR_QUOTA_NOBJS RGW_ATTR_META_PREFIX "quota-count"
119 #define RGW_ATTR_QUOTA_MSIZE RGW_ATTR_META_PREFIX "quota-bytes"
121 /* Static Web Site of Swift API. */
122 #define RGW_ATTR_WEB_INDEX RGW_ATTR_META_PREFIX "web-index"
123 #define RGW_ATTR_WEB_ERROR RGW_ATTR_META_PREFIX "web-error"
124 #define RGW_ATTR_WEB_LISTINGS RGW_ATTR_META_PREFIX "web-listings"
125 #define RGW_ATTR_WEB_LIST_CSS RGW_ATTR_META_PREFIX "web-listings-css"
126 #define RGW_ATTR_SUBDIR_MARKER RGW_ATTR_META_PREFIX "web-directory-type"
128 #define RGW_ATTR_OLH_PREFIX RGW_ATTR_PREFIX "olh."
130 #define RGW_ATTR_OLH_INFO RGW_ATTR_OLH_PREFIX "info"
131 #define RGW_ATTR_OLH_VER RGW_ATTR_OLH_PREFIX "ver"
132 #define RGW_ATTR_OLH_ID_TAG RGW_ATTR_OLH_PREFIX "idtag"
133 #define RGW_ATTR_OLH_PENDING_PREFIX RGW_ATTR_OLH_PREFIX "pending."
135 #define RGW_ATTR_COMPRESSION RGW_ATTR_PREFIX "compression"
137 #define RGW_ATTR_APPEND_PART_NUM RGW_ATTR_PREFIX "append_part_num"
139 /* Attrs to store cloudtier config information. These are used internally
140 * for the replication of cloudtiered objects but not stored as xattrs in
141 * the head object. */
142 #define RGW_ATTR_CLOUD_TIER_TYPE RGW_ATTR_PREFIX "cloud_tier_type"
143 #define RGW_ATTR_CLOUD_TIER_CONFIG RGW_ATTR_PREFIX "cloud_tier_config"
145 #define RGW_ATTR_OBJ_REPLICATION_STATUS RGW_ATTR_PREFIX "amz-replication-status"
146 #define RGW_ATTR_OBJ_REPLICATION_TRACE RGW_ATTR_PREFIX "replication-trace"
149 #define RGW_ATTR_IAM_POLICY RGW_ATTR_PREFIX "iam-policy"
150 #define RGW_ATTR_USER_POLICY RGW_ATTR_PREFIX "user-policy"
151 #define RGW_ATTR_PUBLIC_ACCESS RGW_ATTR_PREFIX "public-access"
153 /* RGW File Attributes */
154 #define RGW_ATTR_UNIX_KEY1 RGW_ATTR_PREFIX "unix-key1"
155 #define RGW_ATTR_UNIX1 RGW_ATTR_PREFIX "unix1"
157 #define RGW_ATTR_CRYPT_PREFIX RGW_ATTR_PREFIX "crypt."
158 #define RGW_ATTR_CRYPT_MODE RGW_ATTR_CRYPT_PREFIX "mode"
159 #define RGW_ATTR_CRYPT_KEYMD5 RGW_ATTR_CRYPT_PREFIX "keymd5"
160 #define RGW_ATTR_CRYPT_KEYID RGW_ATTR_CRYPT_PREFIX "keyid"
161 #define RGW_ATTR_CRYPT_KEYSEL RGW_ATTR_CRYPT_PREFIX "keysel"
162 #define RGW_ATTR_CRYPT_CONTEXT RGW_ATTR_CRYPT_PREFIX "context"
163 #define RGW_ATTR_CRYPT_DATAKEY RGW_ATTR_CRYPT_PREFIX "datakey"
165 /* SSE-S3 Encryption Attributes */
166 #define RGW_ATTR_BUCKET_ENCRYPTION_PREFIX RGW_ATTR_PREFIX "sse-s3."
167 #define RGW_ATTR_BUCKET_ENCRYPTION_POLICY RGW_ATTR_BUCKET_ENCRYPTION_PREFIX "policy"
168 #define RGW_ATTR_BUCKET_ENCRYPTION_KEY_ID RGW_ATTR_BUCKET_ENCRYPTION_PREFIX "key-id"
170 #define RGW_ATTR_TRACE RGW_ATTR_PREFIX "trace"
172 enum class RGWFormat
: int8_t {
180 static inline const char* to_mime_type(const RGWFormat f
)
184 return "application/xml";
186 case RGWFormat::JSON
:
187 return "application/json";
189 case RGWFormat::HTML
:
192 case RGWFormat::PLAIN
:
196 return "invalid format";
200 #define RGW_CAP_READ 0x1
201 #define RGW_CAP_WRITE 0x2
202 #define RGW_CAP_ALL (RGW_CAP_READ | RGW_CAP_WRITE)
204 #define RGW_REST_SWIFT 0x1
205 #define RGW_REST_SWIFT_AUTH 0x2
206 #define RGW_REST_S3 0x4
207 #define RGW_REST_WEBSITE 0x8
208 #define RGW_REST_STS 0x10
209 #define RGW_REST_IAM 0x20
210 #define RGW_REST_SNS 0x30
212 #define RGW_SUSPENDED_USER_AUID (uint64_t)-2
214 #define RGW_OP_TYPE_READ 0x01
215 #define RGW_OP_TYPE_WRITE 0x02
216 #define RGW_OP_TYPE_DELETE 0x04
218 #define RGW_OP_TYPE_MODIFY (RGW_OP_TYPE_WRITE | RGW_OP_TYPE_DELETE)
219 #define RGW_OP_TYPE_ALL (RGW_OP_TYPE_READ | RGW_OP_TYPE_WRITE | RGW_OP_TYPE_DELETE)
221 #define RGW_DEFAULT_MAX_BUCKETS 1000
223 #define RGW_DEFER_TO_BUCKET_ACLS_RECURSE 1
224 #define RGW_DEFER_TO_BUCKET_ACLS_FULL_CONTROL 2
226 #define STATUS_CREATED 1900
227 #define STATUS_ACCEPTED 1901
228 #define STATUS_NO_CONTENT 1902
229 #define STATUS_PARTIAL_CONTENT 1903
230 #define STATUS_REDIRECT 1904
231 #define STATUS_NO_APPLY 1905
232 #define STATUS_APPLIED 1906
234 #define ERR_INVALID_BUCKET_NAME 2000
235 #define ERR_INVALID_OBJECT_NAME 2001
236 #define ERR_NO_SUCH_BUCKET 2002
237 #define ERR_METHOD_NOT_ALLOWED 2003
238 #define ERR_INVALID_DIGEST 2004
239 #define ERR_BAD_DIGEST 2005
240 #define ERR_UNRESOLVABLE_EMAIL 2006
241 #define ERR_INVALID_PART 2007
242 #define ERR_INVALID_PART_ORDER 2008
243 #define ERR_NO_SUCH_UPLOAD 2009
244 #define ERR_REQUEST_TIMEOUT 2010
245 #define ERR_LENGTH_REQUIRED 2011
246 #define ERR_REQUEST_TIME_SKEWED 2012
247 #define ERR_BUCKET_EXISTS 2013
248 #define ERR_BAD_URL 2014
249 #define ERR_PRECONDITION_FAILED 2015
250 #define ERR_NOT_MODIFIED 2016
251 #define ERR_INVALID_UTF8 2017
252 #define ERR_UNPROCESSABLE_ENTITY 2018
253 #define ERR_TOO_LARGE 2019
254 #define ERR_TOO_MANY_BUCKETS 2020
255 #define ERR_INVALID_REQUEST 2021
256 #define ERR_TOO_SMALL 2022
257 #define ERR_NOT_FOUND 2023
258 #define ERR_PERMANENT_REDIRECT 2024
259 #define ERR_LOCKED 2025
260 #define ERR_QUOTA_EXCEEDED 2026
261 #define ERR_SIGNATURE_NO_MATCH 2027
262 #define ERR_INVALID_ACCESS_KEY 2028
263 #define ERR_MALFORMED_XML 2029
264 #define ERR_USER_EXIST 2030
265 #define ERR_NOT_SLO_MANIFEST 2031
266 #define ERR_EMAIL_EXIST 2032
267 #define ERR_KEY_EXIST 2033
268 #define ERR_INVALID_SECRET_KEY 2034
269 #define ERR_INVALID_KEY_TYPE 2035
270 #define ERR_INVALID_CAP 2036
271 #define ERR_INVALID_TENANT_NAME 2037
272 #define ERR_WEBSITE_REDIRECT 2038
273 #define ERR_NO_SUCH_WEBSITE_CONFIGURATION 2039
274 #define ERR_AMZ_CONTENT_SHA256_MISMATCH 2040
275 #define ERR_NO_SUCH_LC 2041
276 #define ERR_NO_SUCH_USER 2042
277 #define ERR_NO_SUCH_SUBUSER 2043
278 #define ERR_MFA_REQUIRED 2044
279 #define ERR_NO_SUCH_CORS_CONFIGURATION 2045
280 #define ERR_NO_SUCH_OBJECT_LOCK_CONFIGURATION 2046
281 #define ERR_INVALID_RETENTION_PERIOD 2047
282 #define ERR_NO_SUCH_BUCKET_ENCRYPTION_CONFIGURATION 2048
283 #define ERR_USER_SUSPENDED 2100
284 #define ERR_INTERNAL_ERROR 2200
285 #define ERR_NOT_IMPLEMENTED 2201
286 #define ERR_SERVICE_UNAVAILABLE 2202
287 #define ERR_ROLE_EXISTS 2203
288 #define ERR_MALFORMED_DOC 2204
289 #define ERR_NO_ROLE_FOUND 2205
290 #define ERR_DELETE_CONFLICT 2206
291 #define ERR_NO_SUCH_BUCKET_POLICY 2207
292 #define ERR_INVALID_LOCATION_CONSTRAINT 2208
293 #define ERR_TAG_CONFLICT 2209
294 #define ERR_INVALID_TAG 2210
295 #define ERR_ZERO_IN_URL 2211
296 #define ERR_MALFORMED_ACL_ERROR 2212
297 #define ERR_ZONEGROUP_DEFAULT_PLACEMENT_MISCONFIGURATION 2213
298 #define ERR_INVALID_ENCRYPTION_ALGORITHM 2214
299 #define ERR_INVALID_CORS_RULES_ERROR 2215
300 #define ERR_NO_CORS_FOUND 2216
301 #define ERR_INVALID_WEBSITE_ROUTING_RULES_ERROR 2217
302 #define ERR_RATE_LIMITED 2218
303 #define ERR_POSITION_NOT_EQUAL_TO_LENGTH 2219
304 #define ERR_OBJECT_NOT_APPENDABLE 2220
305 #define ERR_INVALID_BUCKET_STATE 2221
306 #define ERR_INVALID_OBJECT_STATE 2222
308 #define ERR_BUSY_RESHARDING 2300
309 #define ERR_NO_SUCH_ENTITY 2301
310 #define ERR_LIMIT_EXCEEDED 2302
313 #define ERR_PACKED_POLICY_TOO_LARGE 2400
314 #define ERR_INVALID_IDENTITY_TOKEN 2401
316 #define ERR_NO_SUCH_TAG_SET 2402
319 #define UINT32_MAX (0xffffffffu)
322 typedef void *RGWAccessHandle
;
324 /* Helper class used for RGWHTTPArgs parsing */
327 const std::string str
;
331 explicit NameVal(const std::string
& nv
) : str(nv
) {}
335 std::string
& get_name() { return name
; }
336 std::string
& get_val() { return val
; }
339 /** Stores the XML arguments associated with the HTTP request in req_state*/
341 std::string str
, empty_str
;
342 std::map
<std::string
, std::string
> val_map
;
343 std::map
<std::string
, std::string
> sys_val_map
;
344 std::map
<std::string
, std::string
> sub_resources
;
345 bool has_resp_modifier
= false;
346 bool admin_subresource_added
= false;
348 RGWHTTPArgs() = default;
349 explicit RGWHTTPArgs(const std::string
& s
, const DoutPrefixProvider
*dpp
) {
354 /** Set the arguments; as received */
355 void set(const std::string
& s
) {
356 has_resp_modifier
= false;
358 sub_resources
.clear();
361 /** parse the received arguments */
362 int parse(const DoutPrefixProvider
*dpp
);
363 void append(const std::string
& name
, const std::string
& val
);
364 void remove(const std::string
& name
);
365 /** Get the value for a specific argument parameter */
366 const std::string
& get(const std::string
& name
, bool *exists
= NULL
) const;
367 boost::optional
<const std::string
&>
368 get_optional(const std::string
& name
) const;
369 int get_bool(const std::string
& name
, bool *val
, bool *exists
) const;
370 int get_bool(const char *name
, bool *val
, bool *exists
) const;
371 void get_bool(const char *name
, bool *val
, bool def_val
) const;
372 int get_int(const char *name
, int *val
, int def_val
) const;
374 /** Get the value for specific system argument parameter */
375 std::string
sys_get(const std::string
& name
, bool *exists
= nullptr) const;
377 /** see if a parameter is contained in this RGWHTTPArgs */
378 bool exists(const char *name
) const {
379 return (val_map
.find(name
) != std::end(val_map
));
381 bool sub_resource_exists(const char *name
) const {
382 return (sub_resources
.find(name
) != std::end(sub_resources
));
384 bool exist_obj_excl_sub_resource() const {
385 const char* const obj_sub_resource
[] = {"append", "torrent", "uploadId",
386 "partNumber", "versionId"};
387 for (unsigned i
= 0; i
!= std::size(obj_sub_resource
); i
++) {
388 if (sub_resource_exists(obj_sub_resource
[i
])) return true;
393 std::map
<std::string
, std::string
>& get_params() {
396 const std::map
<std::string
, std::string
>& get_params() const {
399 std::map
<std::string
, std::string
>& get_sys_params() {
402 const std::map
<std::string
, std::string
>& get_sys_params() const {
405 const std::map
<std::string
, std::string
>& get_sub_resources() const {
406 return sub_resources
;
408 unsigned get_num_params() const {
409 return val_map
.size();
411 bool has_response_modifier() const {
412 return has_resp_modifier
;
414 void set_system() { /* make all system params visible */
415 std::map
<std::string
, std::string
>::iterator iter
;
416 for (iter
= sys_val_map
.begin(); iter
!= sys_val_map
.end(); ++iter
) {
417 val_map
[iter
->first
] = iter
->second
;
420 const std::string
& get_str() {
425 const char *rgw_conf_get(const std::map
<std::string
, std::string
, ltstr_nocase
>& conf_map
, const char *name
, const char *def_val
);
426 int rgw_conf_get_int(const std::map
<std::string
, std::string
, ltstr_nocase
>& conf_map
, const char *name
, int def_val
);
427 bool rgw_conf_get_bool(const std::map
<std::string
, std::string
, ltstr_nocase
>& conf_map
, const char *name
, bool def_val
);
434 int enable_usage_log
;
435 uint8_t defer_to_bucket_acls
;
436 void init(CephContext
*cct
);
441 defer_to_bucket_acls(0) {
446 std::map
<std::string
, std::string
, ltstr_nocase
> env_map
;
449 void init(CephContext
*cct
);
450 void init(CephContext
*cct
, char **envp
);
451 void set(std::string name
, std::string val
);
452 const char *get(const char *name
, const char *def_val
= nullptr) const;
453 int get_int(const char *name
, int def_val
= 0) const;
454 bool get_bool(const char *name
, bool def_val
= 0);
455 size_t get_size(const char *name
, size_t def_val
= 0) const;
456 bool exists(const char *name
) const;
457 bool exists_prefix(const char *prefix
) const;
458 void remove(const char *name
);
459 const std::map
<std::string
, std::string
, ltstr_nocase
>& get_map() const { return env_map
; }
460 int get_enable_ops_log() const {
461 return conf
.enable_ops_log
;
464 int get_enable_usage_log() const {
465 return conf
.enable_usage_log
;
468 int get_defer_to_bucket_acls() const {
469 return conf
.defer_to_bucket_acls
;
473 // return true if the connection is secure. this either means that the
474 // connection arrived via ssl, or was forwarded as https by a trusted proxy
475 bool rgw_transport_is_secure(CephContext
*cct
, const RGWEnv
& env
);
488 class RGWAccessControlPolicy
;
491 void encode_json(const char *name
, const obj_version
& v
, Formatter
*f
);
492 void encode_json(const char *name
, const RGWUserCaps
& val
, Formatter
*f
);
494 void decode_json_obj(obj_version
& v
, JSONObj
*obj
);
506 void encode_json(const char *name
, const rgw_placement_rule
& val
, ceph::Formatter
*f
);
507 void decode_json_obj(rgw_placement_rule
& v
, JSONObj
*obj
);
509 inline std::ostream
& operator<<(std::ostream
& out
, const rgw_placement_rule
& rule
) {
510 return out
<< rule
.to_str();
514 struct RGWRateLimitInfo
{
515 int64_t max_write_ops
;
516 int64_t max_read_ops
;
517 int64_t max_write_bytes
;
518 int64_t max_read_bytes
;
519 bool enabled
= false;
521 : max_write_ops(0), max_read_ops(0), max_write_bytes(0), max_read_bytes(0) {}
523 void encode(bufferlist
& bl
) const {
524 ENCODE_START(1, 1, bl
);
525 encode(max_write_ops
, bl
);
526 encode(max_read_ops
, bl
);
527 encode(max_write_bytes
, bl
);
528 encode(max_read_bytes
, bl
);
532 void decode(bufferlist::const_iterator
& bl
) {
534 decode(max_write_ops
,bl
);
535 decode(max_read_ops
, bl
);
536 decode(max_write_bytes
,bl
);
537 decode(max_read_bytes
, bl
);
542 void dump(Formatter
*f
) const;
544 void decode_json(JSONObj
*obj
);
547 WRITE_CLASS_ENCODER(RGWRateLimitInfo
)
552 std::string display_name
;
553 std::string user_email
;
554 std::map
<std::string
, RGWAccessKey
> access_keys
;
555 std::map
<std::string
, RGWAccessKey
> swift_keys
;
556 std::map
<std::string
, RGWSubUser
> subusers
;
563 rgw_placement_rule default_placement
;
564 std::list
<std::string
> placement_tags
;
565 std::map
<int, std::string
> temp_url_keys
;
568 std::set
<std::string
> mfa_ids
;
572 max_buckets(RGW_DEFAULT_MAX_BUCKETS
),
573 op_mask(RGW_OP_TYPE_ALL
),
579 RGWAccessKey
* get_key(const std::string
& access_key
) {
580 if (access_keys
.empty())
583 auto k
= access_keys
.find(access_key
);
584 if (k
== access_keys
.end())
590 void encode(bufferlist
& bl
) const {
591 ENCODE_START(22, 9, bl
);
592 encode((uint64_t)0, bl
); // old auid
593 std::string access_key
;
594 std::string secret_key
;
595 if (!access_keys
.empty()) {
596 std::map
<std::string
, RGWAccessKey
>::const_iterator iter
= access_keys
.begin();
597 const RGWAccessKey
& k
= iter
->second
;
601 encode(access_key
, bl
);
602 encode(secret_key
, bl
);
603 encode(display_name
, bl
);
604 encode(user_email
, bl
);
605 std::string swift_name
;
606 std::string swift_key
;
607 if (!swift_keys
.empty()) {
608 std::map
<std::string
, RGWAccessKey
>::const_iterator iter
= swift_keys
.begin();
609 const RGWAccessKey
& k
= iter
->second
;
613 encode(swift_name
, bl
);
614 encode(swift_key
, bl
);
615 encode(user_id
.id
, bl
);
616 encode(access_keys
, bl
);
617 encode(subusers
, bl
);
618 encode(suspended
, bl
);
619 encode(swift_keys
, bl
);
620 encode(max_buckets
, bl
);
624 encode(default_placement
, bl
);
625 encode(placement_tags
, bl
);
626 encode(quota
.bucket_quota
, bl
);
627 encode(temp_url_keys
, bl
);
628 encode(quota
.user_quota
, bl
);
629 encode(user_id
.tenant
, bl
);
634 std::string assumed_role_arn
; // removed
635 encode(assumed_role_arn
, bl
);
637 encode(user_id
.ns
, bl
);
640 void decode(bufferlist::const_iterator
& bl
) {
641 DECODE_START_LEGACY_COMPAT_LEN_32(22, 9, 9, bl
);
644 decode(old_auid
, bl
);
646 std::string access_key
;
647 std::string secret_key
;
648 decode(access_key
, bl
);
649 decode(secret_key
, bl
);
654 access_keys
[access_key
] = k
;
656 decode(display_name
, bl
);
657 decode(user_email
, bl
);
658 /* We populate swift_keys map later nowadays, but we have to decode. */
659 std::string swift_name
;
660 std::string swift_key
;
661 if (struct_v
>= 3) decode(swift_name
, bl
);
662 if (struct_v
>= 4) decode(swift_key
, bl
);
664 decode(user_id
.id
, bl
);
666 user_id
.id
= access_key
;
668 decode(access_keys
, bl
);
669 decode(subusers
, bl
);
673 decode(suspended
, bl
);
676 decode(swift_keys
, bl
);
678 if (struct_v
>= 10) {
679 decode(max_buckets
, bl
);
681 max_buckets
= RGW_DEFAULT_MAX_BUCKETS
;
683 if (struct_v
>= 11) {
686 if (struct_v
>= 12) {
689 op_mask
= RGW_OP_TYPE_ALL
;
691 if (struct_v
>= 13) {
693 decode(default_placement
, bl
);
694 decode(placement_tags
, bl
); /* tags of allowed placement rules */
696 if (struct_v
>= 14) {
697 decode(quota
.bucket_quota
, bl
);
699 if (struct_v
>= 15) {
700 decode(temp_url_keys
, bl
);
702 if (struct_v
>= 16) {
703 decode(quota
.user_quota
, bl
);
705 if (struct_v
>= 17) {
706 decode(user_id
.tenant
, bl
);
708 user_id
.tenant
.clear();
710 if (struct_v
>= 18) {
713 if (struct_v
>= 19) {
716 if (struct_v
>= 20) {
719 if (struct_v
>= 21) {
720 std::string assumed_role_arn
; // removed
721 decode(assumed_role_arn
, bl
);
723 if (struct_v
>= 22) {
724 decode(user_id
.ns
, bl
);
730 void dump(Formatter
*f
) const;
731 static void generate_test_instances(std::list
<RGWUserInfo
*>& o
);
733 void decode_json(JSONObj
*obj
);
735 WRITE_CLASS_ENCODER(RGWUserInfo
)
737 /// `RGWObjVersionTracker`
738 /// ======================
740 /// What and why is this?
741 /// ---------------------
743 /// This is a wrapper around `cls_version` functionality. If two RGWs
744 /// (or two non-synchronized threads in the same RGW) are accessing
745 /// the same object, they may race and overwrite each other's work.
747 /// This class solves this issue by tracking and recording an object's
748 /// version in the extended attributes. Operations are failed with
749 /// ECANCELED if the version is not what we expect.
754 /// When preparing a read operation, call `prepare_op_for_read`.
755 /// For a write, call `prepare_op_for_write` when preparing the
756 /// operation, and `apply_write` after it succeeds.
758 /// Adhere to the following guidelines:
760 /// - Each RGWObjVersionTracker should be used with only one object.
762 /// - If you receive `ECANCELED`, throw away whatever you were doing
763 /// based on the content of the versioned object, re-read, and
764 /// restart as appropriate.
766 /// - If one code path uses RGWObjVersionTracker, then they all
767 /// should. In a situation where a writer should unconditionally
768 /// overwrite an object, call `generate_new_write_ver` on a default
769 /// constructed `RGWObjVersionTracker`.
771 /// - If we have a version from a previous read, we will check against
772 /// it and fail the read if it doesn't match. Thus, if we want to
773 /// re-read a new version of the object, call `clear()` on the
774 /// `RGWObjVersionTracker`.
776 /// - This type is not thread-safe. Every thread must have its own
779 struct RGWObjVersionTracker
{
780 obj_version read_version
; //< The version read from an object. If
781 // set, this value is used to check the
783 obj_version write_version
; //< Set the object to this version on
786 /// Pointer to the read version.
787 obj_version
* version_for_read() {
788 return &read_version
;
791 /// If we have a write version, return a pointer to it. Otherwise
792 /// return null. This is used in `prepare_op_for_write` to treat the
793 /// `write_version` as effectively an `option` type.
794 obj_version
* version_for_write() {
795 if (write_version
.ver
== 0)
798 return &write_version
;
801 /// If read_version is non-empty, return a pointer to it, otherwise
802 /// null. This is used internally by `prepare_op_for_read` and
803 /// `prepare_op_for_write` to treat the `read_version` as
804 /// effectively an `option` type.
805 obj_version
* version_for_check() {
806 if (read_version
.ver
== 0)
809 return &read_version
;
812 /// This function is to be called on any read operation. If we have
813 /// a non-empty `read_version`, assert on the OSD that the object
814 /// has the same version. Also reads the version into `read_version`.
816 /// This function is defined in `rgw_rados.cc` rather than `rgw_common.cc`.
817 void prepare_op_for_read(librados::ObjectReadOperation
* op
);
819 /// This function is to be called on any write operation. If we have
820 /// a non-empty read operation, assert on the OSD that the object
821 /// has the same version. If we have a non-empty `write_version`,
822 /// set the object to it. Otherwise increment the version on the OSD.
824 /// This function is defined in `rgw_rados.cc` rather than
826 void prepare_op_for_write(librados::ObjectWriteOperation
* op
);
828 /// This function is to be called after the completion of any write
829 /// operation on which `prepare_op_for_write` was called. If we did
830 /// not set the write version explicitly, it increments
831 /// `read_version`. If we did, it sets `read_version` to
832 /// `write_version`. In either case, it clears `write_version`.
834 /// RADOS write operations, at least those not using the relatively
835 /// new RETURNVEC flag, cannot return more information than an error
836 /// code. Thus, write operations can't simply fill in the read
837 /// version the way read operations can, so prepare_op_for_write`
838 /// instructs the OSD to increment the object as stored in RADOS and
839 /// `apply_write` increments our `read_version` in RAM.
841 /// This function is defined in `rgw_rados.cc` rather than
845 /// Clear `read_version` and `write_version`, making the instance
846 /// identical to a default-constructed instance.
848 read_version
= obj_version();
849 write_version
= obj_version();
852 /// Set `write_version` to a new, unique version.
854 /// An `obj_version` contains an opaque, random tag and a
855 /// sequence. If the tags of two `obj_version`s don't match, the
856 /// versions are unordered and unequal. This function creates a
857 /// version with a new tag, ensuring that any other process
858 /// operating on the object will receive `ECANCELED` and will know
859 /// to re-read the object and restart whatever it was doing.
860 void generate_new_write_ver(CephContext
* cct
);
863 inline std::ostream
& operator<<(std::ostream
& out
, const obj_version
&v
)
865 out
<< v
.tag
<< ":" << v
.ver
;
869 inline std::ostream
& operator<<(std::ostream
& out
, const RGWObjVersionTracker
&ot
)
871 out
<< "{r=" << ot
.read_version
<< ",w=" << ot
.write_version
<< "}";
875 enum RGWBucketFlags
{
876 BUCKET_SUSPENDED
= 0x1,
877 BUCKET_VERSIONED
= 0x2,
878 BUCKET_VERSIONS_SUSPENDED
= 0x4,
879 BUCKET_DATASYNC_DISABLED
= 0X8,
880 BUCKET_MFA_ENABLED
= 0X10,
881 BUCKET_OBJ_LOCK_ENABLED
= 0X20,
886 struct RGWBucketInfo
{
890 std::string zonegroup
;
891 ceph::real_time creation_time
;
892 rgw_placement_rule placement_rule
;
893 bool has_instance_obj
{false};
894 RGWObjVersionTracker objv_tracker
; /* we don't need to serialize this, for runtime tracking */
897 // layout of bucket index objects
898 rgw::BucketLayout layout
;
900 // Represents the shard number for blind bucket.
901 const static uint32_t NUM_SHARDS_BLIND_BUCKET
;
903 bool requester_pays
{false};
905 bool has_website
{false};
906 RGWBucketWebsiteConf website_conf
;
908 bool swift_versioning
{false};
909 std::string swift_ver_location
;
911 std::map
<std::string
, uint32_t> mdsearch_config
;
914 cls_rgw_reshard_status reshard_status
{cls_rgw_reshard_status::NOT_RESHARDING
};
915 std::string new_bucket_instance_id
;
917 RGWObjectLock obj_lock
;
919 std::optional
<rgw_sync_policy_info
> sync_policy
;
921 void encode(bufferlist
& bl
) const;
922 void decode(bufferlist::const_iterator
& bl
);
924 void dump(Formatter
*f
) const;
925 static void generate_test_instances(std::list
<RGWBucketInfo
*>& o
);
927 void decode_json(JSONObj
*obj
);
929 bool versioned() const { return (flags
& BUCKET_VERSIONED
) != 0; }
930 int versioning_status() const { return flags
& (BUCKET_VERSIONED
| BUCKET_VERSIONS_SUSPENDED
| BUCKET_MFA_ENABLED
); }
931 bool versioning_enabled() const { return (versioning_status() & (BUCKET_VERSIONED
| BUCKET_VERSIONS_SUSPENDED
)) == BUCKET_VERSIONED
; }
932 bool mfa_enabled() const { return (versioning_status() & BUCKET_MFA_ENABLED
) != 0; }
933 bool datasync_flag_enabled() const { return (flags
& BUCKET_DATASYNC_DISABLED
) == 0; }
934 bool obj_lock_enabled() const { return (flags
& BUCKET_OBJ_LOCK_ENABLED
) != 0; }
936 bool has_swift_versioning() const {
937 /* A bucket may be versioned through one mechanism only. */
938 return swift_versioning
&& !versioned();
941 void set_sync_policy(rgw_sync_policy_info
&& policy
);
943 bool empty_sync_policy() const;
945 bool is_indexless() const {
946 return rgw::is_layout_indexless(layout
.current_index
);
948 const rgw::bucket_index_layout_generation
& get_current_index() const {
949 return layout
.current_index
;
951 rgw::bucket_index_layout_generation
& get_current_index() {
952 return layout
.current_index
;
958 WRITE_CLASS_ENCODER(RGWBucketInfo
)
960 struct RGWBucketEntryPoint
964 ceph::real_time creation_time
;
967 bool has_bucket_info
;
968 RGWBucketInfo old_bucket_info
;
970 RGWBucketEntryPoint() : linked(false), has_bucket_info(false) {}
972 void encode(bufferlist
& bl
) const {
973 ENCODE_START(10, 8, bl
);
975 encode(owner
.id
, bl
);
977 uint64_t ctime
= (uint64_t)real_clock::to_time_t(creation_time
);
980 encode(creation_time
, bl
);
983 void decode(bufferlist::const_iterator
& bl
) {
985 DECODE_START_LEGACY_COMPAT_LEN_32(10, 4, 4, bl
);
987 /* ouch, old entry, contains the bucket info itself */
988 old_bucket_info
.decode(orig_iter
);
989 has_bucket_info
= true;
992 has_bucket_info
= false;
994 decode(owner
.id
, bl
);
999 creation_time
= real_clock::from_time_t((time_t)ctime
);
1001 if (struct_v
>= 9) {
1004 if (struct_v
>= 10) {
1005 decode(creation_time
, bl
);
1010 void dump(Formatter
*f
) const;
1011 void decode_json(JSONObj
*obj
);
1012 static void generate_test_instances(std::list
<RGWBucketEntryPoint
*>& o
);
1014 WRITE_CLASS_ENCODER(RGWBucketEntryPoint
)
1016 struct RGWStorageStats
1018 RGWObjCategory category
;
1020 uint64_t size_rounded
;
1021 uint64_t num_objects
;
1022 uint64_t size_utilized
{0}; //< size after compression, encryption
1023 bool dump_utilized
; // whether dump should include utilized values
1025 RGWStorageStats(bool _dump_utilized
=true)
1026 : category(RGWObjCategory::None
),
1030 dump_utilized(_dump_utilized
)
1033 void dump(Formatter
*f
) const;
1034 }; // RGWStorageStats
1038 /* Namespaced forward declarations. */
1042 class AWSBrowserUploadAbstractor
;
1052 using meta_map_t
= boost::container::flat_map
<std::string
, std::string
>;
1057 meta_map_t x_meta_map
;
1058 meta_map_t crypt_attribute_map
;
1062 std::string script_uri
;
1063 std::string request_uri
;
1064 std::string request_uri_aws4
;
1065 std::string effective_uri
;
1066 std::string request_params
;
1068 std::string storage_class
;
1070 req_info(CephContext
*cct
, const RGWEnv
*env
);
1071 void rebuild_from(req_info
& src
);
1072 void init_meta_info(const DoutPrefixProvider
*dpp
, bool *found_bad_meta
);
1075 struct req_init_state
{
1076 /* Keeps [[tenant]:]bucket until we parse the token. */
1077 std::string url_bucket
;
1078 std::string src_bucket
;
1081 #include "rgw_auth.h"
1085 /** Store all the state necessary to complete and respond to an HTTP request*/
1086 struct req_state
: DoutPrefixProvider
{
1088 const RGWProcessEnv
& penv
;
1089 rgw::io::BasicClient
*cio
{nullptr};
1090 http_op op
{OP_UNKNOWN
};
1091 RGWOpType op_type
{};
1092 std::shared_ptr
<RateLimiter
> ratelimit_data
;
1093 RGWRateLimitInfo user_ratelimit
;
1094 RGWRateLimitInfo bucket_ratelimit
;
1095 std::string ratelimit_bucket_marker
;
1096 std::string ratelimit_user_name
;
1097 bool content_started
{false};
1098 RGWFormat format
{RGWFormat::PLAIN
};
1099 ceph::Formatter
*formatter
{nullptr};
1100 std::string decoded_uri
;
1101 std::string relative_uri
;
1102 const char *length
{nullptr};
1103 int64_t content_length
{0};
1104 std::map
<std::string
, std::string
> generic_attrs
;
1106 bool expect_cont
{false};
1107 uint64_t obj_size
{0};
1108 bool enable_ops_log
;
1109 bool enable_usage_log
;
1110 uint8_t defer_to_bucket_acls
;
1111 uint32_t perm_mask
{0};
1113 /* Set once when url_bucket is parsed and not violated thereafter. */
1114 std::string account_name
;
1116 std::string bucket_tenant
;
1117 std::string bucket_name
;
1119 /* bucket is only created in rgw_build_bucket_policies() and should never be
1121 std::unique_ptr
<rgw::sal::Bucket
> bucket
;
1122 std::unique_ptr
<rgw::sal::Object
> object
;
1123 std::string src_tenant_name
;
1124 std::string src_bucket_name
;
1125 std::unique_ptr
<rgw::sal::Object
> src_object
;
1126 ACLOwner bucket_owner
;
1129 std::string zonegroup_name
;
1130 std::string zonegroup_endpoint
;
1131 std::string bucket_instance_id
;
1132 int bucket_instance_shard_id
{-1};
1133 std::string redirect_zone_endpoint
;
1135 std::string redirect
;
1137 real_time bucket_mtime
;
1138 std::map
<std::string
, ceph::bufferlist
> bucket_attrs
;
1139 bool bucket_exists
{false};
1140 rgw_placement_rule dest_placement
;
1142 bool has_bad_meta
{false};
1144 std::unique_ptr
<rgw::sal::User
> user
;
1147 /* TODO(rzarzynski): switch out to the static_ptr for both members. */
1149 /* Object having the knowledge about an authenticated identity and allowing
1150 * to apply it during the authorization phase (verify_permission() methods
1151 * of a given RGWOp). Thus, it bounds authentication and authorization steps
1152 * through a well-defined interface. For more details, see rgw_auth.h. */
1153 std::unique_ptr
<rgw::auth::Identity
> identity
;
1155 std::shared_ptr
<rgw::auth::Completer
> completer
;
1157 /* A container for credentials of the S3's browser upload. It's necessary
1158 * because: 1) the ::authenticate() method of auth engines and strategies
1159 * take req_state only; 2) auth strategies live much longer than RGWOps -
1160 * there is no way to pass additional data dependencies through ctors. */
1163 friend class RGWPostObj_ObjStore_S3
;
1165 friend class rgw::auth::s3::AWSBrowserUploadAbstractor
;
1166 friend class rgw::auth::s3::STSEngine
;
1168 std::string access_key
;
1169 std::string signature
;
1170 std::string x_amz_algorithm
;
1171 std::string x_amz_credential
;
1172 std::string x_amz_date
;
1173 std::string x_amz_security_token
;
1174 ceph::bufferlist encoded_policy
;
1178 std::unique_ptr
<RGWAccessControlPolicy
> user_acl
;
1179 std::unique_ptr
<RGWAccessControlPolicy
> bucket_acl
;
1180 std::unique_ptr
<RGWAccessControlPolicy
> object_acl
;
1182 rgw::IAM::Environment env
;
1183 boost::optional
<rgw::IAM::Policy
> iam_policy
;
1184 boost::optional
<PublicAccessBlockConfiguration
> bucket_access_conf
;
1185 std::vector
<rgw::IAM::Policy
> iam_user_policies
;
1187 /* Is the request made by an user marked as a system one?
1188 * Being system user means we also have the admin status. */
1189 bool system_request
{false};
1191 std::string canned_acl
;
1192 bool has_acl_header
{false};
1193 bool local_source
{false}; /* source is local */
1197 /* Content-Disposition override for TempURL of Swift API. */
1199 std::string override
;
1200 std::string fallback
;
1203 std::string host_id
;
1206 req_init_state init_state
;
1208 using Clock
= ceph::coarse_real_clock
;
1209 Clock::time_point time
;
1211 Clock::duration
time_elapsed() const { return Clock::now() - time
; }
1213 std::string dialect
;
1215 std::string trans_id
;
1220 bool mfa_verified
{false};
1222 /// optional coroutine context
1223 optional_yield yield
{null_yield
};
1225 //token claims from STS token for ops log (can be used for Keystone token also)
1226 std::vector
<std::string
> token_claims
;
1228 std::vector
<rgw::IAM::Policy
> session_policies
;
1231 bool trace_enabled
= false;
1233 //Principal tags that come in as part of AssumeRoleWithWebIdentity
1234 std::vector
<std::pair
<std::string
, std::string
>> principal_tags
;
1236 req_state(CephContext
* _cct
, const RGWProcessEnv
& penv
, RGWEnv
* e
, uint64_t id
);
1240 void set_user(std::unique_ptr
<rgw::sal::User
>& u
) { user
.swap(u
); }
1241 bool is_err() const { return err
.is_err(); }
1243 // implements DoutPrefixProvider
1244 std::ostream
& gen_prefix(std::ostream
& out
) const override
;
1245 CephContext
* get_cct() const override
{ return cct
; }
1246 unsigned get_subsys() const override
{ return ceph_subsys_rgw
; }
1249 void set_req_state_err(req_state
*, int);
1250 void set_req_state_err(req_state
*, int, const std::string
&);
1251 void set_req_state_err(struct rgw_err
&, int, const int);
1252 void dump(req_state
*);
1254 /** Store basic data on bucket */
1255 struct RGWBucketEnt
{
1258 size_t size_rounded
;
1259 ceph::real_time creation_time
;
1262 /* The placement_rule is necessary to calculate per-storage-policy statics
1263 * of the Swift API. Although the info available in RGWBucketInfo, we need
1264 * to duplicate it here to not affect the performance of buckets listing. */
1265 rgw_placement_rule placement_rule
;
1272 RGWBucketEnt(const RGWBucketEnt
&) = default;
1273 RGWBucketEnt(RGWBucketEnt
&&) = default;
1274 explicit RGWBucketEnt(const rgw_user
& u
, cls_user_bucket_entry
&& e
)
1275 : bucket(u
, std::move(e
.bucket
)),
1277 size_rounded(e
.size_rounded
),
1278 creation_time(e
.creation_time
),
1282 RGWBucketEnt
& operator=(const RGWBucketEnt
&) = default;
1284 void convert(cls_user_bucket_entry
*b
) const {
1285 bucket
.convert(&b
->bucket
);
1287 b
->size_rounded
= size_rounded
;
1288 b
->creation_time
= creation_time
;
1292 void encode(bufferlist
& bl
) const {
1293 ENCODE_START(7, 5, bl
);
1295 __u32 mt
= ceph::real_clock::to_time_t(creation_time
);
1296 std::string empty_str
; // originally had the bucket name here, but we encode bucket later
1297 encode(empty_str
, bl
);
1304 encode(creation_time
, bl
);
1305 encode(placement_rule
, bl
);
1308 void decode(bufferlist::const_iterator
& bl
) {
1309 DECODE_START_LEGACY_COMPAT_LEN(7, 5, 5, bl
);
1312 std::string empty_str
; // backward compatibility
1313 decode(empty_str
, bl
);
1318 creation_time
= ceph::real_clock::from_time_t(mt
);
1328 decode(creation_time
, bl
);
1330 decode(placement_rule
, bl
);
1333 void dump(Formatter
*f
) const;
1334 static void generate_test_instances(std::list
<RGWBucketEnt
*>& o
);
1336 WRITE_CLASS_ENCODER(RGWBucketEnt
)
1338 struct rgw_cache_entry_info
{
1339 std::string cache_locator
;
1342 rgw_cache_entry_info() : gen(0) {}
1345 inline std::ostream
& operator<<(std::ostream
& out
, const rgw_obj
&o
) {
1346 return out
<< o
.bucket
.name
<< ":" << o
.get_oid();
1349 struct multipart_upload_info
1351 rgw_placement_rule dest_placement
;
1353 void encode(bufferlist
& bl
) const {
1354 ENCODE_START(1, 1, bl
);
1355 encode(dest_placement
, bl
);
1359 void decode(bufferlist::const_iterator
& bl
) {
1360 DECODE_START(1, bl
);
1361 decode(dest_placement
, bl
);
1365 WRITE_CLASS_ENCODER(multipart_upload_info
)
1367 static inline void buf_to_hex(const unsigned char* const buf
,
1372 for (size_t i
= 0; i
< len
; i
++) {
1373 ::sprintf(&str
[i
*2], "%02x", static_cast<int>(buf
[i
]));
1377 template<size_t N
> static inline std::array
<char, N
* 2 + 1>
1378 buf_to_hex(const std::array
<unsigned char, N
>& buf
)
1380 static_assert(N
> 0, "The input array must be at least one element long");
1382 std::array
<char, N
* 2 + 1> hex_dest
;
1383 buf_to_hex(buf
.data(), N
, hex_dest
.data());
1387 static inline int hexdigit(char c
)
1389 if (c
>= '0' && c
<= '9')
1392 if (c
>= 'A' && c
<= 'F')
1393 return c
- 'A' + 0xa;
1397 static inline int hex_to_buf(const char *hex
, char *buf
, int len
)
1400 const char *p
= hex
;
1405 int d
= hexdigit(*p
);
1422 static inline int rgw_str_to_bool(const char *s
, int def_val
)
1427 return (strcasecmp(s
, "true") == 0 ||
1428 strcasecmp(s
, "on") == 0 ||
1429 strcasecmp(s
, "yes") == 0 ||
1430 strcasecmp(s
, "1") == 0);
1433 static inline void append_rand_alpha(CephContext
*cct
, const std::string
& src
, std::string
& dest
, int len
)
1437 gen_rand_alphanumeric(cct
, buf
, len
);
1442 static inline uint64_t rgw_rounded_kb(uint64_t bytes
)
1444 return (bytes
+ 1023) / 1024;
1447 static inline uint64_t rgw_rounded_objsize(uint64_t bytes
)
1449 return ((bytes
+ 4095) & ~4095);
1452 static inline uint64_t rgw_rounded_objsize_kb(uint64_t bytes
)
1454 return ((bytes
+ 4095) & ~4095) / 1024;
1457 /* implement combining step, S3 header canonicalization; k is a
1458 * valid header and in lc form */
1459 void rgw_add_amz_meta_header(
1460 meta_map_t
& x_meta_map
,
1461 const std::string
& k
,
1462 const std::string
& v
);
1464 enum rgw_set_action_if_set
{
1465 DISCARD
=0, OVERWRITE
, APPEND
1468 bool rgw_set_amz_meta_header(
1469 meta_map_t
& x_meta_map
,
1470 const std::string
& k
,
1471 const std::string
& v
, rgw_set_action_if_set f
);
1473 extern std::string
rgw_string_unquote(const std::string
& s
);
1474 extern void parse_csv_string(const std::string
& ival
, std::vector
<std::string
>& ovals
);
1475 extern int parse_key_value(std::string
& in_str
, std::string
& key
, std::string
& val
);
1476 extern int parse_key_value(std::string
& in_str
, const char *delim
, std::string
& key
, std::string
& val
);
1478 extern boost::optional
<std::pair
<std::string_view
,std::string_view
>>
1479 parse_key_value(const std::string_view
& in_str
,
1480 const std::string_view
& delim
);
1481 extern boost::optional
<std::pair
<std::string_view
,std::string_view
>>
1482 parse_key_value(const std::string_view
& in_str
);
1484 struct rgw_name_to_flag
{
1485 const char *type_name
;
1490 extern int parse_time(const char *time_str
, real_time
*time
);
1491 extern bool parse_rfc2616(const char *s
, struct tm
*t
);
1492 extern bool parse_iso8601(const char *s
, struct tm
*t
, uint32_t *pns
= NULL
, bool extended_format
= true);
1493 extern std::string
rgw_trim_whitespace(const std::string
& src
);
1494 extern std::string_view
rgw_trim_whitespace(const std::string_view
& src
);
1495 extern std::string
rgw_trim_quotes(const std::string
& val
);
1497 extern void rgw_to_iso8601(const real_time
& t
, char *dest
, int buf_size
);
1498 extern void rgw_to_iso8601(const real_time
& t
, std::string
*dest
);
1499 extern std::string
rgw_to_asctime(const utime_t
& t
);
1501 struct perm_state_base
{
1503 const rgw::IAM::Environment
& env
;
1504 rgw::auth::Identity
*identity
;
1505 const RGWBucketInfo bucket_info
;
1507 bool defer_to_bucket_acls
;
1508 boost::optional
<PublicAccessBlockConfiguration
> bucket_access_conf
;
1510 perm_state_base(CephContext
*_cct
,
1511 const rgw::IAM::Environment
& _env
,
1512 rgw::auth::Identity
*_identity
,
1513 const RGWBucketInfo
& _bucket_info
,
1515 bool _defer_to_bucket_acls
,
1516 boost::optional
<PublicAccessBlockConfiguration
> _bucket_acess_conf
= boost::none
) :
1519 identity(_identity
),
1520 bucket_info(_bucket_info
),
1521 perm_mask(_perm_mask
),
1522 defer_to_bucket_acls(_defer_to_bucket_acls
),
1523 bucket_access_conf(_bucket_acess_conf
)
1526 virtual ~perm_state_base() {}
1528 virtual const char *get_referer() const = 0;
1529 virtual std::optional
<bool> get_request_payer() const = 0; /*
1530 * empty state means that request_payer param was not passed in
1535 struct perm_state
: public perm_state_base
{
1536 const char *referer
;
1539 perm_state(CephContext
*_cct
,
1540 const rgw::IAM::Environment
& _env
,
1541 rgw::auth::Identity
*_identity
,
1542 const RGWBucketInfo
& _bucket_info
,
1544 bool _defer_to_bucket_acls
,
1545 const char *_referer
,
1546 bool _request_payer
) : perm_state_base(_cct
,
1551 _defer_to_bucket_acls
),
1553 request_payer(_request_payer
) {}
1555 const char *get_referer() const override
{
1559 std::optional
<bool> get_request_payer() const override
{
1560 return request_payer
;
1564 /** Check if the req_state's user has the necessary permissions
1565 * to do the requested action */
1566 bool verify_bucket_permission_no_policy(
1567 const DoutPrefixProvider
* dpp
,
1568 struct perm_state_base
* const s
,
1569 RGWAccessControlPolicy
* const user_acl
,
1570 RGWAccessControlPolicy
* const bucket_acl
,
1573 bool verify_user_permission_no_policy(const DoutPrefixProvider
* dpp
,
1574 struct perm_state_base
* const s
,
1575 RGWAccessControlPolicy
* const user_acl
,
1578 bool verify_object_permission_no_policy(const DoutPrefixProvider
* dpp
,
1579 struct perm_state_base
* const s
,
1580 RGWAccessControlPolicy
* const user_acl
,
1581 RGWAccessControlPolicy
* const bucket_acl
,
1582 RGWAccessControlPolicy
* const object_acl
,
1585 /** Check if the req_state's user has the necessary permissions
1586 * to do the requested action */
1587 rgw::IAM::Effect
eval_identity_or_session_policies(const DoutPrefixProvider
* dpp
,
1588 const std::vector
<rgw::IAM::Policy
>& user_policies
,
1589 const rgw::IAM::Environment
& env
,
1591 const rgw::ARN
& arn
);
1592 bool verify_user_permission(const DoutPrefixProvider
* dpp
,
1593 req_state
* const s
,
1594 RGWAccessControlPolicy
* const user_acl
,
1595 const std::vector
<rgw::IAM::Policy
>& user_policies
,
1596 const std::vector
<rgw::IAM::Policy
>& session_policies
,
1597 const rgw::ARN
& res
,
1599 bool mandatory_policy
=true);
1600 bool verify_user_permission_no_policy(const DoutPrefixProvider
* dpp
,
1601 req_state
* const s
,
1602 RGWAccessControlPolicy
* const user_acl
,
1604 bool verify_user_permission(const DoutPrefixProvider
* dpp
,
1605 req_state
* const s
,
1606 const rgw::ARN
& res
,
1608 bool mandatory_policy
=true);
1609 bool verify_user_permission_no_policy(const DoutPrefixProvider
* dpp
,
1610 req_state
* const s
,
1612 bool verify_bucket_permission(
1613 const DoutPrefixProvider
* dpp
,
1614 req_state
* const s
,
1615 const rgw_bucket
& bucket
,
1616 RGWAccessControlPolicy
* const user_acl
,
1617 RGWAccessControlPolicy
* const bucket_acl
,
1618 const boost::optional
<rgw::IAM::Policy
>& bucket_policy
,
1619 const std::vector
<rgw::IAM::Policy
>& identity_policies
,
1620 const std::vector
<rgw::IAM::Policy
>& session_policies
,
1622 bool verify_bucket_permission(const DoutPrefixProvider
* dpp
, req_state
* const s
, const uint64_t op
);
1623 bool verify_bucket_permission_no_policy(
1624 const DoutPrefixProvider
* dpp
,
1625 req_state
* const s
,
1626 RGWAccessControlPolicy
* const user_acl
,
1627 RGWAccessControlPolicy
* const bucket_acl
,
1629 bool verify_bucket_permission_no_policy(const DoutPrefixProvider
* dpp
,
1630 req_state
* const s
,
1632 int verify_bucket_owner_or_policy(req_state
* const s
,
1634 extern bool verify_object_permission(
1635 const DoutPrefixProvider
* dpp
,
1636 req_state
* const s
,
1638 RGWAccessControlPolicy
* const user_acl
,
1639 RGWAccessControlPolicy
* const bucket_acl
,
1640 RGWAccessControlPolicy
* const object_acl
,
1641 const boost::optional
<rgw::IAM::Policy
>& bucket_policy
,
1642 const std::vector
<rgw::IAM::Policy
>& identity_policies
,
1643 const std::vector
<rgw::IAM::Policy
>& session_policies
,
1645 extern bool verify_object_permission(const DoutPrefixProvider
* dpp
, req_state
*s
, uint64_t op
);
1646 extern bool verify_object_permission_no_policy(
1647 const DoutPrefixProvider
* dpp
,
1648 req_state
* const s
,
1649 RGWAccessControlPolicy
* const user_acl
,
1650 RGWAccessControlPolicy
* const bucket_acl
,
1651 RGWAccessControlPolicy
* const object_acl
,
1653 extern bool verify_object_permission_no_policy(const DoutPrefixProvider
* dpp
, req_state
*s
,
1655 extern int verify_object_lock(
1656 const DoutPrefixProvider
* dpp
,
1657 const rgw::sal::Attrs
& attrs
,
1658 const bool bypass_perm
,
1659 const bool bypass_governance_mode
);
1661 /** Convert an input URL into a sane object name
1662 * by converting %-escaped std::strings into characters, etc*/
1663 extern void rgw_uri_escape_char(char c
, std::string
& dst
);
1664 extern std::string
url_decode(const std::string_view
& src_str
,
1665 bool in_query
= false);
1666 extern void url_encode(const std::string
& src
, std::string
& dst
,
1667 bool encode_slash
= true);
1668 extern std::string
url_encode(const std::string
& src
, bool encode_slash
= true);
1669 extern std::string
url_remove_prefix(const std::string
& url
); // Removes hhtp, https and www from url
1670 /* destination should be CEPH_CRYPTO_HMACSHA1_DIGESTSIZE bytes long */
1671 extern void calc_hmac_sha1(const char *key
, int key_len
,
1672 const char *msg
, int msg_len
, char *dest
);
1674 static inline sha1_digest_t
1675 calc_hmac_sha1(const std::string_view
& key
, const std::string_view
& msg
) {
1677 calc_hmac_sha1(key
.data(), key
.size(), msg
.data(), msg
.size(),
1678 reinterpret_cast<char*>(dest
.v
));
1682 /* destination should be CEPH_CRYPTO_HMACSHA256_DIGESTSIZE bytes long */
1683 extern void calc_hmac_sha256(const char *key
, int key_len
,
1684 const char *msg
, int msg_len
,
1687 static inline sha256_digest_t
1688 calc_hmac_sha256(const char *key
, const int key_len
,
1689 const char *msg
, const int msg_len
) {
1690 sha256_digest_t dest
;
1691 calc_hmac_sha256(key
, key_len
, msg
, msg_len
,
1692 reinterpret_cast<char*>(dest
.v
));
1696 static inline sha256_digest_t
1697 calc_hmac_sha256(const std::string_view
& key
, const std::string_view
& msg
) {
1698 sha256_digest_t dest
;
1699 calc_hmac_sha256(key
.data(), key
.size(),
1700 msg
.data(), msg
.size(),
1701 reinterpret_cast<char*>(dest
.v
));
1705 static inline sha256_digest_t
1706 calc_hmac_sha256(const sha256_digest_t
&key
,
1707 const std::string_view
& msg
) {
1708 sha256_digest_t dest
;
1709 calc_hmac_sha256(reinterpret_cast<const char*>(key
.v
), sha256_digest_t::SIZE
,
1710 msg
.data(), msg
.size(),
1711 reinterpret_cast<char*>(dest
.v
));
1715 static inline sha256_digest_t
1716 calc_hmac_sha256(const std::vector
<unsigned char>& key
,
1717 const std::string_view
& msg
) {
1718 sha256_digest_t dest
;
1719 calc_hmac_sha256(reinterpret_cast<const char*>(key
.data()), key
.size(),
1720 msg
.data(), msg
.size(),
1721 reinterpret_cast<char*>(dest
.v
));
1725 template<size_t KeyLenN
>
1726 static inline sha256_digest_t
1727 calc_hmac_sha256(const std::array
<unsigned char, KeyLenN
>& key
,
1728 const std::string_view
& msg
) {
1729 sha256_digest_t dest
;
1730 calc_hmac_sha256(reinterpret_cast<const char*>(key
.data()), key
.size(),
1731 msg
.data(), msg
.size(),
1732 reinterpret_cast<char*>(dest
.v
));
1736 extern sha256_digest_t
calc_hash_sha256(const std::string_view
& msg
);
1738 extern ceph::crypto::SHA256
* calc_hash_sha256_open_stream();
1739 extern void calc_hash_sha256_update_stream(ceph::crypto::SHA256
* hash
,
1742 extern std::string
calc_hash_sha256_close_stream(ceph::crypto::SHA256
** phash
);
1743 extern std::string
calc_hash_sha256_restart_stream(ceph::crypto::SHA256
** phash
);
1745 extern int rgw_parse_op_type_list(const std::string
& str
, uint32_t *perm
);
1747 static constexpr uint32_t MATCH_POLICY_ACTION
= 0x01;
1748 static constexpr uint32_t MATCH_POLICY_RESOURCE
= 0x02;
1749 static constexpr uint32_t MATCH_POLICY_ARN
= 0x04;
1750 static constexpr uint32_t MATCH_POLICY_STRING
= 0x08;
1752 extern bool match_policy(std::string_view pattern
, std::string_view input
,
1755 extern std::string
camelcase_dash_http_attr(const std::string
& orig
);
1756 extern std::string
lowercase_dash_http_attr(const std::string
& orig
);
1758 void rgw_setup_saved_curl_handles();
1759 void rgw_release_all_curl_handles();
1761 static inline void rgw_escape_str(const std::string
& s
, char esc_char
,
1762 char special_char
, std::string
*dest
)
1764 const char *src
= s
.c_str();
1765 char dest_buf
[s
.size() * 2 + 1];
1766 char *destp
= dest_buf
;
1768 for (size_t i
= 0; i
< s
.size(); i
++) {
1770 if (c
== esc_char
|| c
== special_char
) {
1771 *destp
++ = esc_char
;
1779 static inline ssize_t
rgw_unescape_str(const std::string
& s
, ssize_t ofs
,
1780 char esc_char
, char special_char
,
1783 const char *src
= s
.c_str();
1784 char dest_buf
[s
.size() + 1];
1785 char *destp
= dest_buf
;
1790 for (size_t i
= ofs
; i
< s
.size(); i
++) {
1792 if (!esc
&& c
== esc_char
) {
1796 if (!esc
&& c
== special_char
) {
1799 return (ssize_t
)i
+ 1;
1806 return std::string::npos
;
1809 static inline std::string
rgw_bl_str(ceph::buffer::list
& raw
)
1811 size_t len
= raw
.length();
1812 std::string
s(raw
.c_str(), len
);
1813 while (len
&& !s
[len
- 1]) {
1820 template <typename T
>
1821 int decode_bl(bufferlist
& bl
, T
& t
)
1823 auto iter
= bl
.cbegin();
1826 } catch (buffer::error
& err
) {
1832 extern int rgw_bucket_parse_bucket_instance(const std::string
& bucket_instance
, std::string
*bucket_name
, std::string
*bucket_id
, int *shard_id
);
1834 boost::intrusive_ptr
<CephContext
>
1835 rgw_global_init(const std::map
<std::string
,std::string
> *defaults
,
1836 std::vector
< const char* >& args
,
1837 uint32_t module_type
, code_environment_t code_env
,