1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab ft=cpp
5 * Ceph - scalable distributed file system
7 * Copyright (C) 2019 Red Hat, Inc.
9 * This is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU Lesser General Public
11 * License version 2.1, as published by the Free Software
12 * Foundation. See file COPYING.
18 #include "rgw_common.h"
19 #include "rgw_compression_types.h"
26 class rgw_obj_select
{
27 rgw_placement_rule placement_rule
;
33 rgw_obj_select() : is_raw(false) {}
34 explicit rgw_obj_select(const rgw_obj
& _obj
) : obj(_obj
), is_raw(false) {}
35 explicit rgw_obj_select(const rgw_raw_obj
& _raw_obj
) : raw_obj(_raw_obj
), is_raw(true) {}
36 rgw_obj_select(const rgw_obj_select
& rhs
) {
37 placement_rule
= rhs
.placement_rule
;
40 raw_obj
= rhs
.raw_obj
;
46 rgw_raw_obj
get_raw_obj(const RGWZoneGroup
& zonegroup
, const RGWZoneParams
& zone_params
) const;
47 rgw_raw_obj
get_raw_obj(RGWRados
*store
) const;
49 rgw_obj_select
& operator=(const rgw_obj
& rhs
) {
55 rgw_obj_select
& operator=(const rgw_raw_obj
& rhs
) {
61 void set_placement_rule(const rgw_placement_rule
& rule
) {
62 placement_rule
= rule
;
64 void dump(Formatter
*f
) const;
67 struct RGWObjManifestPart
{
68 rgw_obj loc
; /* the object where the data is located */
69 uint64_t loc_ofs
; /* the offset at that object where the data is located */
70 uint64_t size
; /* the part size */
72 RGWObjManifestPart() : loc_ofs(0), size(0) {}
74 void encode(bufferlist
& bl
) const {
75 ENCODE_START(2, 2, bl
);
82 void decode(bufferlist::const_iterator
& bl
) {
83 DECODE_START_LEGACY_COMPAT_LEN_32(2, 2, 2, bl
);
90 void dump(Formatter
*f
) const;
91 static void generate_test_instances(std::list
<RGWObjManifestPart
*>& o
);
93 WRITE_CLASS_ENCODER(RGWObjManifestPart
)
96 The manifest defines a set of rules for structuring the object parts.
97 There are a few terms to note:
98 - head: the head part of the object, which is the part that contains
99 the first chunk of data. An object might not have a head (as in the
100 case of multipart-part objects).
101 - stripe: data portion of a single rgw object that resides on a single
103 - part: a collection of stripes that make a contiguous part of an
104 object. A regular object will only have one part (although might have
105 many stripes), a multipart object might have many parts. Each part
106 has a fixed stripe size, although the last stripe of a part might
107 be smaller than that. Consecutive parts may be merged if their stripe
111 struct RGWObjManifestRule
{
112 uint32_t start_part_num
;
114 uint64_t part_size
; /* each part size, 0 if there's no part size, meaning it's unlimited */
115 uint64_t stripe_max_size
; /* underlying obj max size */
116 string override_prefix
;
118 RGWObjManifestRule() : start_part_num(0), start_ofs(0), part_size(0), stripe_max_size(0) {}
119 RGWObjManifestRule(uint32_t _start_part_num
, uint64_t _start_ofs
, uint64_t _part_size
, uint64_t _stripe_max_size
) :
120 start_part_num(_start_part_num
), start_ofs(_start_ofs
), part_size(_part_size
), stripe_max_size(_stripe_max_size
) {}
122 void encode(bufferlist
& bl
) const {
123 ENCODE_START(2, 1, bl
);
124 encode(start_part_num
, bl
);
125 encode(start_ofs
, bl
);
126 encode(part_size
, bl
);
127 encode(stripe_max_size
, bl
);
128 encode(override_prefix
, bl
);
132 void decode(bufferlist::const_iterator
& bl
) {
134 decode(start_part_num
, bl
);
135 decode(start_ofs
, bl
);
136 decode(part_size
, bl
);
137 decode(stripe_max_size
, bl
);
139 decode(override_prefix
, bl
);
142 void dump(Formatter
*f
) const;
144 WRITE_CLASS_ENCODER(RGWObjManifestRule
)
146 class RGWObjManifest
{
148 bool explicit_objs
{false}; /* really old manifest? */
149 map
<uint64_t, RGWObjManifestPart
> objs
;
151 uint64_t obj_size
{0};
154 uint64_t head_size
{0};
155 rgw_placement_rule head_placement_rule
;
157 uint64_t max_head_size
{0};
159 rgw_bucket_placement tail_placement
; /* might be different than the original bucket,
160 as object might have been copied across pools */
161 map
<uint64_t, RGWObjManifestRule
> rules
;
163 string tail_instance
; /* tail object's instance */
165 void convert_to_explicit(const RGWZoneGroup
& zonegroup
, const RGWZoneParams
& zone_params
);
166 int append_explicit(RGWObjManifest
& m
, const RGWZoneGroup
& zonegroup
, const RGWZoneParams
& zone_params
);
167 void append_rules(RGWObjManifest
& m
, map
<uint64_t, RGWObjManifestRule
>::iterator
& iter
, string
*override_prefix
);
169 void update_iterators() {
171 end_iter
.seek(obj_size
);
175 RGWObjManifest() : begin_iter(this), end_iter(this) {}
176 RGWObjManifest(const RGWObjManifest
& rhs
) {
179 RGWObjManifest
& operator=(const RGWObjManifest
& rhs
) {
180 explicit_objs
= rhs
.explicit_objs
;
182 obj_size
= rhs
.obj_size
;
184 head_size
= rhs
.head_size
;
185 max_head_size
= rhs
.max_head_size
;
187 tail_placement
= rhs
.tail_placement
;
189 tail_instance
= rhs
.tail_instance
;
191 begin_iter
.set_manifest(this);
192 end_iter
.set_manifest(this);
194 begin_iter
.seek(rhs
.begin_iter
.get_ofs());
195 end_iter
.seek(rhs
.end_iter
.get_ofs());
200 map
<uint64_t, RGWObjManifestPart
>& get_explicit_objs() {
205 void set_explicit(uint64_t _size
, map
<uint64_t, RGWObjManifestPart
>& _objs
) {
206 explicit_objs
= true;
211 void get_implicit_location(uint64_t cur_part_id
, uint64_t cur_stripe
, uint64_t ofs
, string
*override_prefix
, rgw_obj_select
*location
);
213 void set_trivial_rule(uint64_t tail_ofs
, uint64_t stripe_max_size
) {
214 RGWObjManifestRule
rule(0, tail_ofs
, 0, stripe_max_size
);
216 max_head_size
= tail_ofs
;
219 void set_multipart_part_rule(uint64_t stripe_max_size
, uint64_t part_num
) {
220 RGWObjManifestRule
rule(0, 0, 0, stripe_max_size
);
221 rule
.start_part_num
= part_num
;
226 void encode(bufferlist
& bl
) const {
227 ENCODE_START(7, 6, bl
);
228 encode(obj_size
, bl
);
230 encode(explicit_objs
, bl
);
232 encode(head_size
, bl
);
233 encode(max_head_size
, bl
);
236 bool encode_tail_bucket
= !(tail_placement
.bucket
== obj
.bucket
);
237 encode(encode_tail_bucket
, bl
);
238 if (encode_tail_bucket
) {
239 encode(tail_placement
.bucket
, bl
);
241 bool encode_tail_instance
= (tail_instance
!= obj
.key
.instance
);
242 encode(encode_tail_instance
, bl
);
243 if (encode_tail_instance
) {
244 encode(tail_instance
, bl
);
246 encode(head_placement_rule
, bl
);
247 encode(tail_placement
.placement_rule
, bl
);
251 void decode(bufferlist::const_iterator
& bl
) {
252 DECODE_START_LEGACY_COMPAT_LEN_32(7, 2, 2, bl
);
253 decode(obj_size
, bl
);
256 decode(explicit_objs
, bl
);
258 decode(head_size
, bl
);
259 decode(max_head_size
, bl
);
263 explicit_objs
= true;
265 map
<uint64_t, RGWObjManifestPart
>::iterator iter
= objs
.begin();
266 obj
= iter
->second
.loc
;
267 head_size
= iter
->second
.size
;
268 max_head_size
= head_size
;
272 if (explicit_objs
&& head_size
> 0 && !objs
.empty()) {
273 /* patch up manifest due to issue 16435:
274 * the first object in the explicit objs list might not be the one we need to access, use the
275 * head object instead if set. This would happen if we had an old object that was created
276 * when the explicit objs manifest was around, and it got copied.
278 rgw_obj
& obj_0
= objs
[0].loc
;
279 if (!obj_0
.get_oid().empty() && obj_0
.key
.ns
.empty()) {
281 objs
[0].size
= head_size
;
287 decode(tail_placement
.bucket
, bl
);
290 decode(need_to_decode
, bl
);
291 if (need_to_decode
) {
292 decode(tail_placement
.bucket
, bl
);
294 tail_placement
.bucket
= obj
.bucket
;
301 decode(tail_instance
, bl
);
304 decode(need_to_decode
, bl
);
305 if (need_to_decode
) {
306 decode(tail_instance
, bl
);
308 tail_instance
= obj
.key
.instance
;
311 } else { // old object created before 'tail_instance' field added to manifest
312 tail_instance
= obj
.key
.instance
;
316 decode(head_placement_rule
, bl
);
317 decode(tail_placement
.placement_rule
, bl
);
324 void dump(Formatter
*f
) const;
325 static void generate_test_instances(list
<RGWObjManifest
*>& o
);
327 int append(RGWObjManifest
& m
, const RGWZoneGroup
& zonegroup
,
328 const RGWZoneParams
& zone_params
);
329 int append(RGWObjManifest
& m
, RGWSI_Zone
*zone_svc
);
331 bool get_rule(uint64_t ofs
, RGWObjManifestRule
*rule
);
336 return rules
.empty();
339 bool has_explicit_objs() {
340 return explicit_objs
;
345 if (objs
.size() == 1) {
346 map
<uint64_t, RGWObjManifestPart
>::iterator iter
= objs
.begin();
347 rgw_obj
& o
= iter
->second
.loc
;
350 return (objs
.size() >= 2);
352 return (obj_size
> head_size
);
355 void set_head(const rgw_placement_rule
& placement_rule
, const rgw_obj
& _o
, uint64_t _s
) {
356 head_placement_rule
= placement_rule
;
360 if (explicit_objs
&& head_size
> 0) {
362 objs
[0].size
= head_size
;
366 const rgw_obj
& get_obj() {
370 void set_tail_placement(const rgw_placement_rule
& placement_rule
, const rgw_bucket
& _b
) {
371 tail_placement
.placement_rule
= placement_rule
;
372 tail_placement
.bucket
= _b
;
375 const rgw_bucket_placement
& get_tail_placement() {
376 return tail_placement
;
379 const rgw_placement_rule
& get_head_placement_rule() {
380 return head_placement_rule
;
383 void set_prefix(const string
& _p
) {
387 const string
& get_prefix() {
391 void set_tail_instance(const string
& _ti
) {
395 const string
& get_tail_instance() {
396 return tail_instance
;
399 void set_head_size(uint64_t _s
) {
403 void set_obj_size(uint64_t s
) {
409 uint64_t get_obj_size() {
413 uint64_t get_head_size() {
417 uint64_t get_max_head_size() {
418 return max_head_size
;
422 RGWObjManifest
*manifest
= nullptr;
423 uint64_t part_ofs
= 0; /* where current part starts */
424 uint64_t stripe_ofs
= 0; /* where current stripe starts */
425 uint64_t ofs
= 0; /* current position within the object */
426 uint64_t stripe_size
= 0; /* current part size */
430 string cur_override_prefix
;
432 rgw_obj_select location
;
434 map
<uint64_t, RGWObjManifestRule
>::iterator rule_iter
;
435 map
<uint64_t, RGWObjManifestRule
>::iterator next_rule_iter
;
437 map
<uint64_t, RGWObjManifestPart
>::iterator explicit_iter
;
439 void update_explicit_pos();
444 void set_manifest(RGWObjManifest
*m
) {
449 obj_iterator() = default;
450 explicit obj_iterator(RGWObjManifest
*_m
)
451 : obj_iterator(_m
, 0)
453 obj_iterator(RGWObjManifest
*_m
, uint64_t _ofs
) : manifest(_m
) {
456 void seek(uint64_t ofs
);
459 bool operator==(const obj_iterator
& rhs
) const {
460 return (ofs
== rhs
.ofs
);
462 bool operator!=(const obj_iterator
& rhs
) const {
463 return (ofs
!= rhs
.ofs
);
465 const rgw_obj_select
& get_location() {
469 /* start of current stripe */
470 uint64_t get_stripe_ofs() {
471 if (manifest
->explicit_objs
) {
472 return explicit_iter
->first
;
477 /* current ofs relative to start of rgw object */
478 uint64_t get_ofs() const {
483 int get_cur_stripe() const {
487 /* current stripe size */
488 uint64_t get_stripe_size() {
489 if (manifest
->explicit_objs
) {
490 return explicit_iter
->second
.size
;
495 /* offset where data starts within current stripe */
496 uint64_t location_ofs() {
497 if (manifest
->explicit_objs
) {
498 return explicit_iter
->second
.loc_ofs
;
500 return 0; /* all stripes start at zero offset */
503 void update_location();
505 friend class RGWObjManifest
;
506 void dump(Formatter
*f
) const;
507 }; // class obj_iterator
509 const obj_iterator
& obj_begin();
510 const obj_iterator
& obj_end();
511 obj_iterator
obj_find(uint64_t ofs
);
513 obj_iterator begin_iter
;
514 obj_iterator end_iter
;
517 * simple object generator. Using a simple single rule manifest.
520 RGWObjManifest
*manifest
;
522 uint64_t cur_part_ofs
;
525 uint64_t cur_stripe_size
;
530 rgw_obj_select cur_obj
;
532 RGWObjManifestRule rule
;
535 generator() : manifest(NULL
), last_ofs(0), cur_part_ofs(0), cur_part_id(0),
536 cur_stripe(0), cur_stripe_size(0) {}
537 int create_begin(CephContext
*cct
, RGWObjManifest
*manifest
,
538 const rgw_placement_rule
& head_placement_rule
,
539 const rgw_placement_rule
*tail_placement_rule
,
540 const rgw_bucket
& bucket
,
543 int create_next(uint64_t ofs
);
545 rgw_raw_obj
get_cur_obj(RGWZoneGroup
& zonegroup
, RGWZoneParams
& zone_params
) { return cur_obj
.get_raw_obj(zonegroup
, zone_params
); }
546 rgw_raw_obj
get_cur_obj(RGWRados
*store
) const { return cur_obj
.get_raw_obj(store
); }
548 /* total max size of current stripe (including head obj) */
549 uint64_t cur_stripe_max_size() const {
550 return cur_stripe_size
;
554 WRITE_CLASS_ENCODER(RGWObjManifest
)