1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab ft=cpp
5 * Ceph - scalable distributed file system
7 * Copyright (C) 2019 Red Hat, Inc.
9 * This is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU Lesser General Public
11 * License version 2.1, as published by the Free Software
12 * Foundation. See file COPYING.
18 #include "rgw_common.h"
19 #include "rgw_compression_types.h"
27 class rgw_obj_select
{
28 rgw_placement_rule placement_rule
;
34 rgw_obj_select() : is_raw(false) {}
35 explicit rgw_obj_select(const rgw_obj
& _obj
) : obj(_obj
), is_raw(false) {}
36 explicit rgw_obj_select(const rgw_raw_obj
& _raw_obj
) : raw_obj(_raw_obj
), is_raw(true) {}
37 rgw_obj_select(const rgw_obj_select
& rhs
) {
38 placement_rule
= rhs
.placement_rule
;
41 raw_obj
= rhs
.raw_obj
;
47 rgw_raw_obj
get_raw_obj(const RGWZoneGroup
& zonegroup
, const RGWZoneParams
& zone_params
) const;
48 rgw_raw_obj
get_raw_obj(rgw::sal::RGWStore
* store
) const;
50 rgw_obj_select
& operator=(const rgw_obj
& rhs
) {
56 rgw_obj_select
& operator=(const rgw_raw_obj
& rhs
) {
62 void set_placement_rule(const rgw_placement_rule
& rule
) {
63 placement_rule
= rule
;
65 void dump(Formatter
*f
) const;
68 struct RGWObjManifestPart
{
69 rgw_obj loc
; /* the object where the data is located */
70 uint64_t loc_ofs
; /* the offset at that object where the data is located */
71 uint64_t size
; /* the part size */
73 RGWObjManifestPart() : loc_ofs(0), size(0) {}
75 void encode(bufferlist
& bl
) const {
76 ENCODE_START(2, 2, bl
);
83 void decode(bufferlist::const_iterator
& bl
) {
84 DECODE_START_LEGACY_COMPAT_LEN_32(2, 2, 2, bl
);
91 void dump(Formatter
*f
) const;
92 static void generate_test_instances(std::list
<RGWObjManifestPart
*>& o
);
94 WRITE_CLASS_ENCODER(RGWObjManifestPart
)
97 The manifest defines a set of rules for structuring the object parts.
98 There are a few terms to note:
99 - head: the head part of the object, which is the part that contains
100 the first chunk of data. An object might not have a head (as in the
101 case of multipart-part objects).
102 - stripe: data portion of a single rgw object that resides on a single
104 - part: a collection of stripes that make a contiguous part of an
105 object. A regular object will only have one part (although might have
106 many stripes), a multipart object might have many parts. Each part
107 has a fixed stripe size, although the last stripe of a part might
108 be smaller than that. Consecutive parts may be merged if their stripe
112 struct RGWObjManifestRule
{
113 uint32_t start_part_num
;
115 uint64_t part_size
; /* each part size, 0 if there's no part size, meaning it's unlimited */
116 uint64_t stripe_max_size
; /* underlying obj max size */
117 string override_prefix
;
119 RGWObjManifestRule() : start_part_num(0), start_ofs(0), part_size(0), stripe_max_size(0) {}
120 RGWObjManifestRule(uint32_t _start_part_num
, uint64_t _start_ofs
, uint64_t _part_size
, uint64_t _stripe_max_size
) :
121 start_part_num(_start_part_num
), start_ofs(_start_ofs
), part_size(_part_size
), stripe_max_size(_stripe_max_size
) {}
123 void encode(bufferlist
& bl
) const {
124 ENCODE_START(2, 1, bl
);
125 encode(start_part_num
, bl
);
126 encode(start_ofs
, bl
);
127 encode(part_size
, bl
);
128 encode(stripe_max_size
, bl
);
129 encode(override_prefix
, bl
);
133 void decode(bufferlist::const_iterator
& bl
) {
135 decode(start_part_num
, bl
);
136 decode(start_ofs
, bl
);
137 decode(part_size
, bl
);
138 decode(stripe_max_size
, bl
);
140 decode(override_prefix
, bl
);
143 void dump(Formatter
*f
) const;
145 WRITE_CLASS_ENCODER(RGWObjManifestRule
)
147 class RGWObjManifest
{
149 bool explicit_objs
{false}; /* really old manifest? */
150 map
<uint64_t, RGWObjManifestPart
> objs
;
152 uint64_t obj_size
{0};
155 uint64_t head_size
{0};
156 rgw_placement_rule head_placement_rule
;
158 uint64_t max_head_size
{0};
160 rgw_bucket_placement tail_placement
; /* might be different than the original bucket,
161 as object might have been copied across pools */
162 map
<uint64_t, RGWObjManifestRule
> rules
;
164 string tail_instance
; /* tail object's instance */
166 void convert_to_explicit(const DoutPrefixProvider
*dpp
, const RGWZoneGroup
& zonegroup
, const RGWZoneParams
& zone_params
);
167 int append_explicit(const DoutPrefixProvider
*dpp
, RGWObjManifest
& m
, const RGWZoneGroup
& zonegroup
, const RGWZoneParams
& zone_params
);
168 void append_rules(RGWObjManifest
& m
, map
<uint64_t, RGWObjManifestRule
>::iterator
& iter
, string
*override_prefix
);
172 RGWObjManifest() = default;
173 RGWObjManifest(const RGWObjManifest
& rhs
) {
176 RGWObjManifest
& operator=(const RGWObjManifest
& rhs
) {
177 explicit_objs
= rhs
.explicit_objs
;
179 obj_size
= rhs
.obj_size
;
181 head_size
= rhs
.head_size
;
182 max_head_size
= rhs
.max_head_size
;
184 tail_placement
= rhs
.tail_placement
;
186 tail_instance
= rhs
.tail_instance
;
190 map
<uint64_t, RGWObjManifestPart
>& get_explicit_objs() {
195 void set_explicit(uint64_t _size
, map
<uint64_t, RGWObjManifestPart
>& _objs
) {
196 explicit_objs
= true;
201 void get_implicit_location(uint64_t cur_part_id
, uint64_t cur_stripe
, uint64_t ofs
,
202 string
*override_prefix
, rgw_obj_select
*location
) const;
204 void set_trivial_rule(uint64_t tail_ofs
, uint64_t stripe_max_size
) {
205 RGWObjManifestRule
rule(0, tail_ofs
, 0, stripe_max_size
);
207 max_head_size
= tail_ofs
;
210 void set_multipart_part_rule(uint64_t stripe_max_size
, uint64_t part_num
) {
211 RGWObjManifestRule
rule(0, 0, 0, stripe_max_size
);
212 rule
.start_part_num
= part_num
;
217 void encode(bufferlist
& bl
) const {
218 ENCODE_START(7, 6, bl
);
219 encode(obj_size
, bl
);
221 encode(explicit_objs
, bl
);
223 encode(head_size
, bl
);
224 encode(max_head_size
, bl
);
227 bool encode_tail_bucket
= !(tail_placement
.bucket
== obj
.bucket
);
228 encode(encode_tail_bucket
, bl
);
229 if (encode_tail_bucket
) {
230 encode(tail_placement
.bucket
, bl
);
232 bool encode_tail_instance
= (tail_instance
!= obj
.key
.instance
);
233 encode(encode_tail_instance
, bl
);
234 if (encode_tail_instance
) {
235 encode(tail_instance
, bl
);
237 encode(head_placement_rule
, bl
);
238 encode(tail_placement
.placement_rule
, bl
);
242 void decode(bufferlist::const_iterator
& bl
) {
243 DECODE_START_LEGACY_COMPAT_LEN_32(7, 2, 2, bl
);
244 decode(obj_size
, bl
);
247 decode(explicit_objs
, bl
);
249 decode(head_size
, bl
);
250 decode(max_head_size
, bl
);
254 explicit_objs
= true;
256 map
<uint64_t, RGWObjManifestPart
>::iterator iter
= objs
.begin();
257 obj
= iter
->second
.loc
;
258 head_size
= iter
->second
.size
;
259 max_head_size
= head_size
;
263 if (explicit_objs
&& head_size
> 0 && !objs
.empty()) {
264 /* patch up manifest due to issue 16435:
265 * the first object in the explicit objs list might not be the one we need to access, use the
266 * head object instead if set. This would happen if we had an old object that was created
267 * when the explicit objs manifest was around, and it got copied.
269 rgw_obj
& obj_0
= objs
[0].loc
;
270 if (!obj_0
.get_oid().empty() && obj_0
.key
.ns
.empty()) {
272 objs
[0].size
= head_size
;
278 decode(tail_placement
.bucket
, bl
);
281 decode(need_to_decode
, bl
);
282 if (need_to_decode
) {
283 decode(tail_placement
.bucket
, bl
);
285 tail_placement
.bucket
= obj
.bucket
;
292 decode(tail_instance
, bl
);
295 decode(need_to_decode
, bl
);
296 if (need_to_decode
) {
297 decode(tail_instance
, bl
);
299 tail_instance
= obj
.key
.instance
;
302 } else { // old object created before 'tail_instance' field added to manifest
303 tail_instance
= obj
.key
.instance
;
307 decode(head_placement_rule
, bl
);
308 decode(tail_placement
.placement_rule
, bl
);
314 void dump(Formatter
*f
) const;
315 static void generate_test_instances(list
<RGWObjManifest
*>& o
);
317 int append(const DoutPrefixProvider
*dpp
, RGWObjManifest
& m
, const RGWZoneGroup
& zonegroup
,
318 const RGWZoneParams
& zone_params
);
319 int append(const DoutPrefixProvider
*dpp
, RGWObjManifest
& m
, RGWSI_Zone
*zone_svc
);
321 bool get_rule(uint64_t ofs
, RGWObjManifestRule
*rule
);
326 return rules
.empty();
329 bool has_explicit_objs() const {
330 return explicit_objs
;
333 bool has_tail() const {
335 if (objs
.size() == 1) {
336 auto iter
= objs
.begin();
337 const rgw_obj
& o
= iter
->second
.loc
;
340 return (objs
.size() >= 2);
342 return (obj_size
> head_size
);
345 void set_head(const rgw_placement_rule
& placement_rule
, const rgw_obj
& _o
, uint64_t _s
) {
346 head_placement_rule
= placement_rule
;
350 if (explicit_objs
&& head_size
> 0) {
352 objs
[0].size
= head_size
;
356 const rgw_obj
& get_obj() const {
360 void set_tail_placement(const rgw_placement_rule
& placement_rule
, const rgw_bucket
& _b
) {
361 tail_placement
.placement_rule
= placement_rule
;
362 tail_placement
.bucket
= _b
;
365 const rgw_bucket_placement
& get_tail_placement() const {
366 return tail_placement
;
369 const rgw_placement_rule
& get_head_placement_rule() const {
370 return head_placement_rule
;
373 void set_prefix(const string
& _p
) {
377 const string
& get_prefix() const {
381 void set_tail_instance(const string
& _ti
) {
385 const string
& get_tail_instance() const {
386 return tail_instance
;
389 void set_head_size(uint64_t _s
) {
393 void set_obj_size(uint64_t s
) {
397 uint64_t get_obj_size() const {
401 uint64_t get_head_size() const {
405 uint64_t get_max_head_size() const {
406 return max_head_size
;
410 const DoutPrefixProvider
*dpp
;
411 const RGWObjManifest
*manifest
= nullptr;
412 uint64_t part_ofs
= 0; /* where current part starts */
413 uint64_t stripe_ofs
= 0; /* where current stripe starts */
414 uint64_t ofs
= 0; /* current position within the object */
415 uint64_t stripe_size
= 0; /* current part size */
419 string cur_override_prefix
;
421 rgw_obj_select location
;
423 map
<uint64_t, RGWObjManifestRule
>::const_iterator rule_iter
;
424 map
<uint64_t, RGWObjManifestRule
>::const_iterator next_rule_iter
;
425 map
<uint64_t, RGWObjManifestPart
>::const_iterator explicit_iter
;
427 void update_explicit_pos();
430 obj_iterator() = default;
431 explicit obj_iterator(const DoutPrefixProvider
*_dpp
, const RGWObjManifest
*_m
)
432 : obj_iterator(_dpp
, _m
, 0)
434 obj_iterator(const DoutPrefixProvider
*_dpp
, const RGWObjManifest
*_m
, uint64_t _ofs
) : dpp(_dpp
), manifest(_m
) {
437 void seek(uint64_t ofs
);
440 bool operator==(const obj_iterator
& rhs
) const {
441 return (ofs
== rhs
.ofs
);
443 bool operator!=(const obj_iterator
& rhs
) const {
444 return (ofs
!= rhs
.ofs
);
446 const rgw_obj_select
& get_location() {
450 /* where current part starts */
451 uint64_t get_part_ofs() const {
455 /* start of current stripe */
456 uint64_t get_stripe_ofs() {
457 if (manifest
->explicit_objs
) {
458 return explicit_iter
->first
;
463 /* current ofs relative to start of rgw object */
464 uint64_t get_ofs() const {
469 int get_cur_stripe() const {
473 /* current stripe size */
474 uint64_t get_stripe_size() {
475 if (manifest
->explicit_objs
) {
476 return explicit_iter
->second
.size
;
481 /* offset where data starts within current stripe */
482 uint64_t location_ofs() {
483 if (manifest
->explicit_objs
) {
484 return explicit_iter
->second
.loc_ofs
;
486 return 0; /* all stripes start at zero offset */
489 void update_location();
491 void dump(Formatter
*f
) const;
492 }; // class obj_iterator
494 obj_iterator
obj_begin(const DoutPrefixProvider
*dpp
) const { return obj_iterator
{dpp
, this}; }
495 obj_iterator
obj_end(const DoutPrefixProvider
*dpp
) const { return obj_iterator
{dpp
, this, obj_size
}; }
496 obj_iterator
obj_find(const DoutPrefixProvider
*dpp
, uint64_t ofs
) const {
497 return obj_iterator
{dpp
, this, std::min(ofs
, obj_size
)};
501 * simple object generator. Using a simple single rule manifest.
504 RGWObjManifest
*manifest
;
506 uint64_t cur_part_ofs
;
509 uint64_t cur_stripe_size
;
514 rgw_obj_select cur_obj
;
516 RGWObjManifestRule rule
;
519 generator() : manifest(NULL
), last_ofs(0), cur_part_ofs(0), cur_part_id(0),
520 cur_stripe(0), cur_stripe_size(0) {}
521 int create_begin(CephContext
*cct
, RGWObjManifest
*manifest
,
522 const rgw_placement_rule
& head_placement_rule
,
523 const rgw_placement_rule
*tail_placement_rule
,
524 const rgw_bucket
& bucket
,
527 int create_next(uint64_t ofs
);
529 rgw_raw_obj
get_cur_obj(RGWZoneGroup
& zonegroup
, RGWZoneParams
& zone_params
) { return cur_obj
.get_raw_obj(zonegroup
, zone_params
); }
530 rgw_raw_obj
get_cur_obj(rgw::sal::RGWStore
* store
) const { return cur_obj
.get_raw_obj(store
); }
532 /* total max size of current stripe (including head obj) */
533 uint64_t cur_stripe_max_size() const {
534 return cur_stripe_size
;
538 WRITE_CLASS_ENCODER(RGWObjManifest
)