]>
Commit | Line | Data |
---|---|---|
9f95a23c TL |
1 | // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- |
2 | // vim: ts=8 sw=2 smarttab ft=cpp | |
3 | ||
4 | /* | |
5 | * Ceph - scalable distributed file system | |
6 | * | |
7 | * Copyright (C) 2019 Red Hat, Inc. | |
8 | * | |
9 | * This is free software; you can redistribute it and/or | |
10 | * modify it under the terms of the GNU Lesser General Public | |
11 | * License version 2.1, as published by the Free Software | |
12 | * Foundation. See file COPYING. | |
13 | * | |
14 | */ | |
15 | ||
16 | #pragma once | |
17 | ||
18 | #include "rgw_common.h" | |
19 | #include "rgw_compression_types.h" | |
f67539c2 | 20 | #include "rgw_sal.h" |
9f95a23c TL |
21 | |
22 | class RGWSI_Zone; | |
23 | struct RGWZoneGroup; | |
24 | struct RGWZoneParams; | |
25 | class RGWRados; | |
26 | ||
27 | class rgw_obj_select { | |
28 | rgw_placement_rule placement_rule; | |
29 | rgw_obj obj; | |
30 | rgw_raw_obj raw_obj; | |
31 | bool is_raw; | |
32 | ||
33 | public: | |
34 | rgw_obj_select() : is_raw(false) {} | |
35 | explicit rgw_obj_select(const rgw_obj& _obj) : obj(_obj), is_raw(false) {} | |
36 | explicit rgw_obj_select(const rgw_raw_obj& _raw_obj) : raw_obj(_raw_obj), is_raw(true) {} | |
37 | rgw_obj_select(const rgw_obj_select& rhs) { | |
38 | placement_rule = rhs.placement_rule; | |
39 | is_raw = rhs.is_raw; | |
40 | if (is_raw) { | |
41 | raw_obj = rhs.raw_obj; | |
42 | } else { | |
43 | obj = rhs.obj; | |
44 | } | |
45 | } | |
46 | ||
47 | rgw_raw_obj get_raw_obj(const RGWZoneGroup& zonegroup, const RGWZoneParams& zone_params) const; | |
f67539c2 | 48 | rgw_raw_obj get_raw_obj(rgw::sal::RGWStore* store) const; |
9f95a23c TL |
49 | |
50 | rgw_obj_select& operator=(const rgw_obj& rhs) { | |
51 | obj = rhs; | |
52 | is_raw = false; | |
53 | return *this; | |
54 | } | |
55 | ||
56 | rgw_obj_select& operator=(const rgw_raw_obj& rhs) { | |
57 | raw_obj = rhs; | |
58 | is_raw = true; | |
59 | return *this; | |
60 | } | |
61 | ||
62 | void set_placement_rule(const rgw_placement_rule& rule) { | |
63 | placement_rule = rule; | |
64 | } | |
65 | void dump(Formatter *f) const; | |
66 | }; | |
67 | ||
68 | struct RGWObjManifestPart { | |
69 | rgw_obj loc; /* the object where the data is located */ | |
70 | uint64_t loc_ofs; /* the offset at that object where the data is located */ | |
71 | uint64_t size; /* the part size */ | |
72 | ||
73 | RGWObjManifestPart() : loc_ofs(0), size(0) {} | |
74 | ||
75 | void encode(bufferlist& bl) const { | |
76 | ENCODE_START(2, 2, bl); | |
77 | encode(loc, bl); | |
78 | encode(loc_ofs, bl); | |
79 | encode(size, bl); | |
80 | ENCODE_FINISH(bl); | |
81 | } | |
82 | ||
83 | void decode(bufferlist::const_iterator& bl) { | |
84 | DECODE_START_LEGACY_COMPAT_LEN_32(2, 2, 2, bl); | |
85 | decode(loc, bl); | |
86 | decode(loc_ofs, bl); | |
87 | decode(size, bl); | |
88 | DECODE_FINISH(bl); | |
89 | } | |
90 | ||
91 | void dump(Formatter *f) const; | |
92 | static void generate_test_instances(std::list<RGWObjManifestPart*>& o); | |
93 | }; | |
94 | WRITE_CLASS_ENCODER(RGWObjManifestPart) | |
95 | ||
96 | /* | |
97 | The manifest defines a set of rules for structuring the object parts. | |
98 | There are a few terms to note: | |
99 | - head: the head part of the object, which is the part that contains | |
100 | the first chunk of data. An object might not have a head (as in the | |
101 | case of multipart-part objects). | |
102 | - stripe: data portion of a single rgw object that resides on a single | |
103 | rados object. | |
104 | - part: a collection of stripes that make a contiguous part of an | |
105 | object. A regular object will only have one part (although might have | |
106 | many stripes), a multipart object might have many parts. Each part | |
107 | has a fixed stripe size, although the last stripe of a part might | |
108 | be smaller than that. Consecutive parts may be merged if their stripe | |
109 | value is the same. | |
110 | */ | |
111 | ||
112 | struct RGWObjManifestRule { | |
113 | uint32_t start_part_num; | |
114 | uint64_t start_ofs; | |
115 | uint64_t part_size; /* each part size, 0 if there's no part size, meaning it's unlimited */ | |
116 | uint64_t stripe_max_size; /* underlying obj max size */ | |
117 | string override_prefix; | |
118 | ||
119 | RGWObjManifestRule() : start_part_num(0), start_ofs(0), part_size(0), stripe_max_size(0) {} | |
120 | RGWObjManifestRule(uint32_t _start_part_num, uint64_t _start_ofs, uint64_t _part_size, uint64_t _stripe_max_size) : | |
121 | start_part_num(_start_part_num), start_ofs(_start_ofs), part_size(_part_size), stripe_max_size(_stripe_max_size) {} | |
122 | ||
123 | void encode(bufferlist& bl) const { | |
124 | ENCODE_START(2, 1, bl); | |
125 | encode(start_part_num, bl); | |
126 | encode(start_ofs, bl); | |
127 | encode(part_size, bl); | |
128 | encode(stripe_max_size, bl); | |
129 | encode(override_prefix, bl); | |
130 | ENCODE_FINISH(bl); | |
131 | } | |
132 | ||
133 | void decode(bufferlist::const_iterator& bl) { | |
134 | DECODE_START(2, bl); | |
135 | decode(start_part_num, bl); | |
136 | decode(start_ofs, bl); | |
137 | decode(part_size, bl); | |
138 | decode(stripe_max_size, bl); | |
139 | if (struct_v >= 2) | |
140 | decode(override_prefix, bl); | |
141 | DECODE_FINISH(bl); | |
142 | } | |
143 | void dump(Formatter *f) const; | |
144 | }; | |
145 | WRITE_CLASS_ENCODER(RGWObjManifestRule) | |
146 | ||
147 | class RGWObjManifest { | |
148 | protected: | |
149 | bool explicit_objs{false}; /* really old manifest? */ | |
150 | map<uint64_t, RGWObjManifestPart> objs; | |
151 | ||
152 | uint64_t obj_size{0}; | |
153 | ||
154 | rgw_obj obj; | |
155 | uint64_t head_size{0}; | |
156 | rgw_placement_rule head_placement_rule; | |
157 | ||
158 | uint64_t max_head_size{0}; | |
159 | string prefix; | |
160 | rgw_bucket_placement tail_placement; /* might be different than the original bucket, | |
161 | as object might have been copied across pools */ | |
162 | map<uint64_t, RGWObjManifestRule> rules; | |
163 | ||
164 | string tail_instance; /* tail object's instance */ | |
165 | ||
b3b6e05e TL |
166 | void convert_to_explicit(const DoutPrefixProvider *dpp, const RGWZoneGroup& zonegroup, const RGWZoneParams& zone_params); |
167 | int append_explicit(const DoutPrefixProvider *dpp, RGWObjManifest& m, const RGWZoneGroup& zonegroup, const RGWZoneParams& zone_params); | |
9f95a23c TL |
168 | void append_rules(RGWObjManifest& m, map<uint64_t, RGWObjManifestRule>::iterator& iter, string *override_prefix); |
169 | ||
9f95a23c TL |
170 | public: |
171 | ||
b3b6e05e | 172 | RGWObjManifest() = default; |
9f95a23c TL |
173 | RGWObjManifest(const RGWObjManifest& rhs) { |
174 | *this = rhs; | |
175 | } | |
176 | RGWObjManifest& operator=(const RGWObjManifest& rhs) { | |
177 | explicit_objs = rhs.explicit_objs; | |
178 | objs = rhs.objs; | |
179 | obj_size = rhs.obj_size; | |
180 | obj = rhs.obj; | |
181 | head_size = rhs.head_size; | |
182 | max_head_size = rhs.max_head_size; | |
183 | prefix = rhs.prefix; | |
184 | tail_placement = rhs.tail_placement; | |
185 | rules = rhs.rules; | |
186 | tail_instance = rhs.tail_instance; | |
9f95a23c TL |
187 | return *this; |
188 | } | |
189 | ||
190 | map<uint64_t, RGWObjManifestPart>& get_explicit_objs() { | |
191 | return objs; | |
192 | } | |
193 | ||
194 | ||
195 | void set_explicit(uint64_t _size, map<uint64_t, RGWObjManifestPart>& _objs) { | |
196 | explicit_objs = true; | |
197 | objs.swap(_objs); | |
198 | set_obj_size(_size); | |
199 | } | |
200 | ||
b3b6e05e TL |
201 | void get_implicit_location(uint64_t cur_part_id, uint64_t cur_stripe, uint64_t ofs, |
202 | string *override_prefix, rgw_obj_select *location) const; | |
9f95a23c TL |
203 | |
204 | void set_trivial_rule(uint64_t tail_ofs, uint64_t stripe_max_size) { | |
205 | RGWObjManifestRule rule(0, tail_ofs, 0, stripe_max_size); | |
206 | rules[0] = rule; | |
207 | max_head_size = tail_ofs; | |
208 | } | |
209 | ||
210 | void set_multipart_part_rule(uint64_t stripe_max_size, uint64_t part_num) { | |
211 | RGWObjManifestRule rule(0, 0, 0, stripe_max_size); | |
212 | rule.start_part_num = part_num; | |
213 | rules[0] = rule; | |
214 | max_head_size = 0; | |
215 | } | |
216 | ||
217 | void encode(bufferlist& bl) const { | |
218 | ENCODE_START(7, 6, bl); | |
219 | encode(obj_size, bl); | |
220 | encode(objs, bl); | |
221 | encode(explicit_objs, bl); | |
222 | encode(obj, bl); | |
223 | encode(head_size, bl); | |
224 | encode(max_head_size, bl); | |
225 | encode(prefix, bl); | |
226 | encode(rules, bl); | |
227 | bool encode_tail_bucket = !(tail_placement.bucket == obj.bucket); | |
228 | encode(encode_tail_bucket, bl); | |
229 | if (encode_tail_bucket) { | |
230 | encode(tail_placement.bucket, bl); | |
231 | } | |
232 | bool encode_tail_instance = (tail_instance != obj.key.instance); | |
233 | encode(encode_tail_instance, bl); | |
234 | if (encode_tail_instance) { | |
235 | encode(tail_instance, bl); | |
236 | } | |
237 | encode(head_placement_rule, bl); | |
238 | encode(tail_placement.placement_rule, bl); | |
239 | ENCODE_FINISH(bl); | |
240 | } | |
241 | ||
242 | void decode(bufferlist::const_iterator& bl) { | |
243 | DECODE_START_LEGACY_COMPAT_LEN_32(7, 2, 2, bl); | |
244 | decode(obj_size, bl); | |
245 | decode(objs, bl); | |
246 | if (struct_v >= 3) { | |
247 | decode(explicit_objs, bl); | |
248 | decode(obj, bl); | |
249 | decode(head_size, bl); | |
250 | decode(max_head_size, bl); | |
251 | decode(prefix, bl); | |
252 | decode(rules, bl); | |
253 | } else { | |
254 | explicit_objs = true; | |
255 | if (!objs.empty()) { | |
256 | map<uint64_t, RGWObjManifestPart>::iterator iter = objs.begin(); | |
257 | obj = iter->second.loc; | |
258 | head_size = iter->second.size; | |
259 | max_head_size = head_size; | |
260 | } | |
261 | } | |
262 | ||
263 | if (explicit_objs && head_size > 0 && !objs.empty()) { | |
264 | /* patch up manifest due to issue 16435: | |
265 | * the first object in the explicit objs list might not be the one we need to access, use the | |
266 | * head object instead if set. This would happen if we had an old object that was created | |
267 | * when the explicit objs manifest was around, and it got copied. | |
268 | */ | |
269 | rgw_obj& obj_0 = objs[0].loc; | |
270 | if (!obj_0.get_oid().empty() && obj_0.key.ns.empty()) { | |
271 | objs[0].loc = obj; | |
272 | objs[0].size = head_size; | |
273 | } | |
274 | } | |
275 | ||
276 | if (struct_v >= 4) { | |
277 | if (struct_v < 6) { | |
278 | decode(tail_placement.bucket, bl); | |
279 | } else { | |
280 | bool need_to_decode; | |
281 | decode(need_to_decode, bl); | |
282 | if (need_to_decode) { | |
283 | decode(tail_placement.bucket, bl); | |
284 | } else { | |
285 | tail_placement.bucket = obj.bucket; | |
286 | } | |
287 | } | |
288 | } | |
289 | ||
290 | if (struct_v >= 5) { | |
291 | if (struct_v < 6) { | |
292 | decode(tail_instance, bl); | |
293 | } else { | |
294 | bool need_to_decode; | |
295 | decode(need_to_decode, bl); | |
296 | if (need_to_decode) { | |
297 | decode(tail_instance, bl); | |
298 | } else { | |
299 | tail_instance = obj.key.instance; | |
300 | } | |
301 | } | |
302 | } else { // old object created before 'tail_instance' field added to manifest | |
303 | tail_instance = obj.key.instance; | |
304 | } | |
305 | ||
306 | if (struct_v >= 7) { | |
307 | decode(head_placement_rule, bl); | |
308 | decode(tail_placement.placement_rule, bl); | |
309 | } | |
310 | ||
9f95a23c TL |
311 | DECODE_FINISH(bl); |
312 | } | |
313 | ||
314 | void dump(Formatter *f) const; | |
315 | static void generate_test_instances(list<RGWObjManifest*>& o); | |
316 | ||
b3b6e05e | 317 | int append(const DoutPrefixProvider *dpp, RGWObjManifest& m, const RGWZoneGroup& zonegroup, |
9f95a23c | 318 | const RGWZoneParams& zone_params); |
b3b6e05e | 319 | int append(const DoutPrefixProvider *dpp, RGWObjManifest& m, RGWSI_Zone *zone_svc); |
9f95a23c TL |
320 | |
321 | bool get_rule(uint64_t ofs, RGWObjManifestRule *rule); | |
322 | ||
b3b6e05e | 323 | bool empty() const { |
9f95a23c TL |
324 | if (explicit_objs) |
325 | return objs.empty(); | |
326 | return rules.empty(); | |
327 | } | |
328 | ||
b3b6e05e | 329 | bool has_explicit_objs() const { |
9f95a23c TL |
330 | return explicit_objs; |
331 | } | |
332 | ||
b3b6e05e | 333 | bool has_tail() const { |
9f95a23c TL |
334 | if (explicit_objs) { |
335 | if (objs.size() == 1) { | |
b3b6e05e TL |
336 | auto iter = objs.begin(); |
337 | const rgw_obj& o = iter->second.loc; | |
9f95a23c TL |
338 | return !(obj == o); |
339 | } | |
340 | return (objs.size() >= 2); | |
341 | } | |
342 | return (obj_size > head_size); | |
343 | } | |
344 | ||
345 | void set_head(const rgw_placement_rule& placement_rule, const rgw_obj& _o, uint64_t _s) { | |
346 | head_placement_rule = placement_rule; | |
347 | obj = _o; | |
348 | head_size = _s; | |
349 | ||
350 | if (explicit_objs && head_size > 0) { | |
351 | objs[0].loc = obj; | |
352 | objs[0].size = head_size; | |
353 | } | |
354 | } | |
355 | ||
b3b6e05e | 356 | const rgw_obj& get_obj() const { |
9f95a23c TL |
357 | return obj; |
358 | } | |
359 | ||
360 | void set_tail_placement(const rgw_placement_rule& placement_rule, const rgw_bucket& _b) { | |
361 | tail_placement.placement_rule = placement_rule; | |
362 | tail_placement.bucket = _b; | |
363 | } | |
364 | ||
b3b6e05e | 365 | const rgw_bucket_placement& get_tail_placement() const { |
9f95a23c TL |
366 | return tail_placement; |
367 | } | |
368 | ||
b3b6e05e | 369 | const rgw_placement_rule& get_head_placement_rule() const { |
9f95a23c TL |
370 | return head_placement_rule; |
371 | } | |
372 | ||
373 | void set_prefix(const string& _p) { | |
374 | prefix = _p; | |
375 | } | |
376 | ||
b3b6e05e | 377 | const string& get_prefix() const { |
9f95a23c TL |
378 | return prefix; |
379 | } | |
380 | ||
381 | void set_tail_instance(const string& _ti) { | |
382 | tail_instance = _ti; | |
383 | } | |
384 | ||
b3b6e05e | 385 | const string& get_tail_instance() const { |
9f95a23c TL |
386 | return tail_instance; |
387 | } | |
388 | ||
389 | void set_head_size(uint64_t _s) { | |
390 | head_size = _s; | |
391 | } | |
392 | ||
393 | void set_obj_size(uint64_t s) { | |
394 | obj_size = s; | |
9f95a23c TL |
395 | } |
396 | ||
b3b6e05e | 397 | uint64_t get_obj_size() const { |
9f95a23c TL |
398 | return obj_size; |
399 | } | |
400 | ||
b3b6e05e | 401 | uint64_t get_head_size() const { |
9f95a23c TL |
402 | return head_size; |
403 | } | |
404 | ||
b3b6e05e | 405 | uint64_t get_max_head_size() const { |
9f95a23c TL |
406 | return max_head_size; |
407 | } | |
408 | ||
409 | class obj_iterator { | |
b3b6e05e TL |
410 | const DoutPrefixProvider *dpp; |
411 | const RGWObjManifest *manifest = nullptr; | |
9f95a23c TL |
412 | uint64_t part_ofs = 0; /* where current part starts */ |
413 | uint64_t stripe_ofs = 0; /* where current stripe starts */ | |
414 | uint64_t ofs = 0; /* current position within the object */ | |
415 | uint64_t stripe_size = 0; /* current part size */ | |
416 | ||
417 | int cur_part_id = 0; | |
418 | int cur_stripe = 0; | |
419 | string cur_override_prefix; | |
420 | ||
421 | rgw_obj_select location; | |
422 | ||
b3b6e05e TL |
423 | map<uint64_t, RGWObjManifestRule>::const_iterator rule_iter; |
424 | map<uint64_t, RGWObjManifestRule>::const_iterator next_rule_iter; | |
425 | map<uint64_t, RGWObjManifestPart>::const_iterator explicit_iter; | |
9f95a23c TL |
426 | |
427 | void update_explicit_pos(); | |
428 | ||
9f95a23c TL |
429 | public: |
430 | obj_iterator() = default; | |
b3b6e05e TL |
431 | explicit obj_iterator(const DoutPrefixProvider *_dpp, const RGWObjManifest *_m) |
432 | : obj_iterator(_dpp, _m, 0) | |
9f95a23c | 433 | {} |
b3b6e05e | 434 | obj_iterator(const DoutPrefixProvider *_dpp, const RGWObjManifest *_m, uint64_t _ofs) : dpp(_dpp), manifest(_m) { |
9f95a23c TL |
435 | seek(_ofs); |
436 | } | |
437 | void seek(uint64_t ofs); | |
438 | ||
439 | void operator++(); | |
e306af50 | 440 | bool operator==(const obj_iterator& rhs) const { |
9f95a23c TL |
441 | return (ofs == rhs.ofs); |
442 | } | |
e306af50 | 443 | bool operator!=(const obj_iterator& rhs) const { |
9f95a23c TL |
444 | return (ofs != rhs.ofs); |
445 | } | |
446 | const rgw_obj_select& get_location() { | |
447 | return location; | |
448 | } | |
449 | ||
adb31ebb TL |
450 | /* where current part starts */ |
451 | uint64_t get_part_ofs() const { | |
452 | return part_ofs; | |
453 | } | |
454 | ||
9f95a23c TL |
455 | /* start of current stripe */ |
456 | uint64_t get_stripe_ofs() { | |
457 | if (manifest->explicit_objs) { | |
458 | return explicit_iter->first; | |
459 | } | |
460 | return stripe_ofs; | |
461 | } | |
462 | ||
463 | /* current ofs relative to start of rgw object */ | |
464 | uint64_t get_ofs() const { | |
465 | return ofs; | |
466 | } | |
467 | ||
468 | /* stripe number */ | |
469 | int get_cur_stripe() const { | |
470 | return cur_stripe; | |
471 | } | |
472 | ||
473 | /* current stripe size */ | |
474 | uint64_t get_stripe_size() { | |
475 | if (manifest->explicit_objs) { | |
476 | return explicit_iter->second.size; | |
477 | } | |
478 | return stripe_size; | |
479 | } | |
480 | ||
481 | /* offset where data starts within current stripe */ | |
482 | uint64_t location_ofs() { | |
483 | if (manifest->explicit_objs) { | |
484 | return explicit_iter->second.loc_ofs; | |
485 | } | |
486 | return 0; /* all stripes start at zero offset */ | |
487 | } | |
488 | ||
489 | void update_location(); | |
490 | ||
9f95a23c | 491 | void dump(Formatter *f) const; |
e306af50 | 492 | }; // class obj_iterator |
9f95a23c | 493 | |
b3b6e05e TL |
494 | obj_iterator obj_begin(const DoutPrefixProvider *dpp) const { return obj_iterator{dpp, this}; } |
495 | obj_iterator obj_end(const DoutPrefixProvider *dpp) const { return obj_iterator{dpp, this, obj_size}; } | |
496 | obj_iterator obj_find(const DoutPrefixProvider *dpp, uint64_t ofs) const { | |
497 | return obj_iterator{dpp, this, std::min(ofs, obj_size)}; | |
498 | } | |
9f95a23c TL |
499 | |
500 | /* | |
501 | * simple object generator. Using a simple single rule manifest. | |
502 | */ | |
503 | class generator { | |
504 | RGWObjManifest *manifest; | |
505 | uint64_t last_ofs; | |
506 | uint64_t cur_part_ofs; | |
507 | int cur_part_id; | |
508 | int cur_stripe; | |
509 | uint64_t cur_stripe_size; | |
510 | string cur_oid; | |
511 | ||
512 | string oid_prefix; | |
513 | ||
514 | rgw_obj_select cur_obj; | |
515 | ||
516 | RGWObjManifestRule rule; | |
517 | ||
518 | public: | |
519 | generator() : manifest(NULL), last_ofs(0), cur_part_ofs(0), cur_part_id(0), | |
520 | cur_stripe(0), cur_stripe_size(0) {} | |
521 | int create_begin(CephContext *cct, RGWObjManifest *manifest, | |
522 | const rgw_placement_rule& head_placement_rule, | |
523 | const rgw_placement_rule *tail_placement_rule, | |
524 | const rgw_bucket& bucket, | |
525 | const rgw_obj& obj); | |
526 | ||
527 | int create_next(uint64_t ofs); | |
528 | ||
529 | rgw_raw_obj get_cur_obj(RGWZoneGroup& zonegroup, RGWZoneParams& zone_params) { return cur_obj.get_raw_obj(zonegroup, zone_params); } | |
f67539c2 | 530 | rgw_raw_obj get_cur_obj(rgw::sal::RGWStore* store) const { return cur_obj.get_raw_obj(store); } |
9f95a23c TL |
531 | |
532 | /* total max size of current stripe (including head obj) */ | |
533 | uint64_t cur_stripe_max_size() const { | |
534 | return cur_stripe_size; | |
535 | } | |
536 | }; | |
537 | }; | |
538 | WRITE_CLASS_ENCODER(RGWObjManifest) |