]>
Commit | Line | Data |
---|---|---|
9f95a23c TL |
1 | // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- |
2 | // vim: ts=8 sw=2 smarttab ft=cpp | |
3 | ||
4 | /* | |
5 | * Ceph - scalable distributed file system | |
6 | * | |
7 | * Copyright (C) 2019 Red Hat, Inc. | |
8 | * | |
9 | * This is free software; you can redistribute it and/or | |
10 | * modify it under the terms of the GNU Lesser General Public | |
11 | * License version 2.1, as published by the Free Software | |
12 | * Foundation. See file COPYING. | |
13 | * | |
14 | */ | |
15 | ||
16 | #pragma once | |
17 | ||
18 | #include "rgw_common.h" | |
19 | #include "rgw_compression_types.h" | |
20 | ||
21 | class RGWSI_Zone; | |
22 | struct RGWZoneGroup; | |
23 | struct RGWZoneParams; | |
24 | class RGWRados; | |
25 | ||
26 | class rgw_obj_select { | |
27 | rgw_placement_rule placement_rule; | |
28 | rgw_obj obj; | |
29 | rgw_raw_obj raw_obj; | |
30 | bool is_raw; | |
31 | ||
32 | public: | |
33 | rgw_obj_select() : is_raw(false) {} | |
34 | explicit rgw_obj_select(const rgw_obj& _obj) : obj(_obj), is_raw(false) {} | |
35 | explicit rgw_obj_select(const rgw_raw_obj& _raw_obj) : raw_obj(_raw_obj), is_raw(true) {} | |
36 | rgw_obj_select(const rgw_obj_select& rhs) { | |
37 | placement_rule = rhs.placement_rule; | |
38 | is_raw = rhs.is_raw; | |
39 | if (is_raw) { | |
40 | raw_obj = rhs.raw_obj; | |
41 | } else { | |
42 | obj = rhs.obj; | |
43 | } | |
44 | } | |
45 | ||
46 | rgw_raw_obj get_raw_obj(const RGWZoneGroup& zonegroup, const RGWZoneParams& zone_params) const; | |
47 | rgw_raw_obj get_raw_obj(RGWRados *store) const; | |
48 | ||
49 | rgw_obj_select& operator=(const rgw_obj& rhs) { | |
50 | obj = rhs; | |
51 | is_raw = false; | |
52 | return *this; | |
53 | } | |
54 | ||
55 | rgw_obj_select& operator=(const rgw_raw_obj& rhs) { | |
56 | raw_obj = rhs; | |
57 | is_raw = true; | |
58 | return *this; | |
59 | } | |
60 | ||
61 | void set_placement_rule(const rgw_placement_rule& rule) { | |
62 | placement_rule = rule; | |
63 | } | |
64 | void dump(Formatter *f) const; | |
65 | }; | |
66 | ||
67 | struct RGWObjManifestPart { | |
68 | rgw_obj loc; /* the object where the data is located */ | |
69 | uint64_t loc_ofs; /* the offset at that object where the data is located */ | |
70 | uint64_t size; /* the part size */ | |
71 | ||
72 | RGWObjManifestPart() : loc_ofs(0), size(0) {} | |
73 | ||
74 | void encode(bufferlist& bl) const { | |
75 | ENCODE_START(2, 2, bl); | |
76 | encode(loc, bl); | |
77 | encode(loc_ofs, bl); | |
78 | encode(size, bl); | |
79 | ENCODE_FINISH(bl); | |
80 | } | |
81 | ||
82 | void decode(bufferlist::const_iterator& bl) { | |
83 | DECODE_START_LEGACY_COMPAT_LEN_32(2, 2, 2, bl); | |
84 | decode(loc, bl); | |
85 | decode(loc_ofs, bl); | |
86 | decode(size, bl); | |
87 | DECODE_FINISH(bl); | |
88 | } | |
89 | ||
90 | void dump(Formatter *f) const; | |
91 | static void generate_test_instances(std::list<RGWObjManifestPart*>& o); | |
92 | }; | |
93 | WRITE_CLASS_ENCODER(RGWObjManifestPart) | |
94 | ||
95 | /* | |
96 | The manifest defines a set of rules for structuring the object parts. | |
97 | There are a few terms to note: | |
98 | - head: the head part of the object, which is the part that contains | |
99 | the first chunk of data. An object might not have a head (as in the | |
100 | case of multipart-part objects). | |
101 | - stripe: data portion of a single rgw object that resides on a single | |
102 | rados object. | |
103 | - part: a collection of stripes that make a contiguous part of an | |
104 | object. A regular object will only have one part (although might have | |
105 | many stripes), a multipart object might have many parts. Each part | |
106 | has a fixed stripe size, although the last stripe of a part might | |
107 | be smaller than that. Consecutive parts may be merged if their stripe | |
108 | value is the same. | |
109 | */ | |
110 | ||
111 | struct RGWObjManifestRule { | |
112 | uint32_t start_part_num; | |
113 | uint64_t start_ofs; | |
114 | uint64_t part_size; /* each part size, 0 if there's no part size, meaning it's unlimited */ | |
115 | uint64_t stripe_max_size; /* underlying obj max size */ | |
116 | string override_prefix; | |
117 | ||
118 | RGWObjManifestRule() : start_part_num(0), start_ofs(0), part_size(0), stripe_max_size(0) {} | |
119 | RGWObjManifestRule(uint32_t _start_part_num, uint64_t _start_ofs, uint64_t _part_size, uint64_t _stripe_max_size) : | |
120 | start_part_num(_start_part_num), start_ofs(_start_ofs), part_size(_part_size), stripe_max_size(_stripe_max_size) {} | |
121 | ||
122 | void encode(bufferlist& bl) const { | |
123 | ENCODE_START(2, 1, bl); | |
124 | encode(start_part_num, bl); | |
125 | encode(start_ofs, bl); | |
126 | encode(part_size, bl); | |
127 | encode(stripe_max_size, bl); | |
128 | encode(override_prefix, bl); | |
129 | ENCODE_FINISH(bl); | |
130 | } | |
131 | ||
132 | void decode(bufferlist::const_iterator& bl) { | |
133 | DECODE_START(2, bl); | |
134 | decode(start_part_num, bl); | |
135 | decode(start_ofs, bl); | |
136 | decode(part_size, bl); | |
137 | decode(stripe_max_size, bl); | |
138 | if (struct_v >= 2) | |
139 | decode(override_prefix, bl); | |
140 | DECODE_FINISH(bl); | |
141 | } | |
142 | void dump(Formatter *f) const; | |
143 | }; | |
144 | WRITE_CLASS_ENCODER(RGWObjManifestRule) | |
145 | ||
146 | class RGWObjManifest { | |
147 | protected: | |
148 | bool explicit_objs{false}; /* really old manifest? */ | |
149 | map<uint64_t, RGWObjManifestPart> objs; | |
150 | ||
151 | uint64_t obj_size{0}; | |
152 | ||
153 | rgw_obj obj; | |
154 | uint64_t head_size{0}; | |
155 | rgw_placement_rule head_placement_rule; | |
156 | ||
157 | uint64_t max_head_size{0}; | |
158 | string prefix; | |
159 | rgw_bucket_placement tail_placement; /* might be different than the original bucket, | |
160 | as object might have been copied across pools */ | |
161 | map<uint64_t, RGWObjManifestRule> rules; | |
162 | ||
163 | string tail_instance; /* tail object's instance */ | |
164 | ||
165 | void convert_to_explicit(const RGWZoneGroup& zonegroup, const RGWZoneParams& zone_params); | |
166 | int append_explicit(RGWObjManifest& m, const RGWZoneGroup& zonegroup, const RGWZoneParams& zone_params); | |
167 | void append_rules(RGWObjManifest& m, map<uint64_t, RGWObjManifestRule>::iterator& iter, string *override_prefix); | |
168 | ||
169 | void update_iterators() { | |
170 | begin_iter.seek(0); | |
171 | end_iter.seek(obj_size); | |
172 | } | |
173 | public: | |
174 | ||
175 | RGWObjManifest() : begin_iter(this), end_iter(this) {} | |
176 | RGWObjManifest(const RGWObjManifest& rhs) { | |
177 | *this = rhs; | |
178 | } | |
179 | RGWObjManifest& operator=(const RGWObjManifest& rhs) { | |
180 | explicit_objs = rhs.explicit_objs; | |
181 | objs = rhs.objs; | |
182 | obj_size = rhs.obj_size; | |
183 | obj = rhs.obj; | |
184 | head_size = rhs.head_size; | |
185 | max_head_size = rhs.max_head_size; | |
186 | prefix = rhs.prefix; | |
187 | tail_placement = rhs.tail_placement; | |
188 | rules = rhs.rules; | |
189 | tail_instance = rhs.tail_instance; | |
190 | ||
191 | begin_iter.set_manifest(this); | |
192 | end_iter.set_manifest(this); | |
193 | ||
194 | begin_iter.seek(rhs.begin_iter.get_ofs()); | |
195 | end_iter.seek(rhs.end_iter.get_ofs()); | |
196 | ||
197 | return *this; | |
198 | } | |
199 | ||
200 | map<uint64_t, RGWObjManifestPart>& get_explicit_objs() { | |
201 | return objs; | |
202 | } | |
203 | ||
204 | ||
205 | void set_explicit(uint64_t _size, map<uint64_t, RGWObjManifestPart>& _objs) { | |
206 | explicit_objs = true; | |
207 | objs.swap(_objs); | |
208 | set_obj_size(_size); | |
209 | } | |
210 | ||
211 | void get_implicit_location(uint64_t cur_part_id, uint64_t cur_stripe, uint64_t ofs, string *override_prefix, rgw_obj_select *location); | |
212 | ||
213 | void set_trivial_rule(uint64_t tail_ofs, uint64_t stripe_max_size) { | |
214 | RGWObjManifestRule rule(0, tail_ofs, 0, stripe_max_size); | |
215 | rules[0] = rule; | |
216 | max_head_size = tail_ofs; | |
217 | } | |
218 | ||
219 | void set_multipart_part_rule(uint64_t stripe_max_size, uint64_t part_num) { | |
220 | RGWObjManifestRule rule(0, 0, 0, stripe_max_size); | |
221 | rule.start_part_num = part_num; | |
222 | rules[0] = rule; | |
223 | max_head_size = 0; | |
224 | } | |
225 | ||
226 | void encode(bufferlist& bl) const { | |
227 | ENCODE_START(7, 6, bl); | |
228 | encode(obj_size, bl); | |
229 | encode(objs, bl); | |
230 | encode(explicit_objs, bl); | |
231 | encode(obj, bl); | |
232 | encode(head_size, bl); | |
233 | encode(max_head_size, bl); | |
234 | encode(prefix, bl); | |
235 | encode(rules, bl); | |
236 | bool encode_tail_bucket = !(tail_placement.bucket == obj.bucket); | |
237 | encode(encode_tail_bucket, bl); | |
238 | if (encode_tail_bucket) { | |
239 | encode(tail_placement.bucket, bl); | |
240 | } | |
241 | bool encode_tail_instance = (tail_instance != obj.key.instance); | |
242 | encode(encode_tail_instance, bl); | |
243 | if (encode_tail_instance) { | |
244 | encode(tail_instance, bl); | |
245 | } | |
246 | encode(head_placement_rule, bl); | |
247 | encode(tail_placement.placement_rule, bl); | |
248 | ENCODE_FINISH(bl); | |
249 | } | |
250 | ||
251 | void decode(bufferlist::const_iterator& bl) { | |
252 | DECODE_START_LEGACY_COMPAT_LEN_32(7, 2, 2, bl); | |
253 | decode(obj_size, bl); | |
254 | decode(objs, bl); | |
255 | if (struct_v >= 3) { | |
256 | decode(explicit_objs, bl); | |
257 | decode(obj, bl); | |
258 | decode(head_size, bl); | |
259 | decode(max_head_size, bl); | |
260 | decode(prefix, bl); | |
261 | decode(rules, bl); | |
262 | } else { | |
263 | explicit_objs = true; | |
264 | if (!objs.empty()) { | |
265 | map<uint64_t, RGWObjManifestPart>::iterator iter = objs.begin(); | |
266 | obj = iter->second.loc; | |
267 | head_size = iter->second.size; | |
268 | max_head_size = head_size; | |
269 | } | |
270 | } | |
271 | ||
272 | if (explicit_objs && head_size > 0 && !objs.empty()) { | |
273 | /* patch up manifest due to issue 16435: | |
274 | * the first object in the explicit objs list might not be the one we need to access, use the | |
275 | * head object instead if set. This would happen if we had an old object that was created | |
276 | * when the explicit objs manifest was around, and it got copied. | |
277 | */ | |
278 | rgw_obj& obj_0 = objs[0].loc; | |
279 | if (!obj_0.get_oid().empty() && obj_0.key.ns.empty()) { | |
280 | objs[0].loc = obj; | |
281 | objs[0].size = head_size; | |
282 | } | |
283 | } | |
284 | ||
285 | if (struct_v >= 4) { | |
286 | if (struct_v < 6) { | |
287 | decode(tail_placement.bucket, bl); | |
288 | } else { | |
289 | bool need_to_decode; | |
290 | decode(need_to_decode, bl); | |
291 | if (need_to_decode) { | |
292 | decode(tail_placement.bucket, bl); | |
293 | } else { | |
294 | tail_placement.bucket = obj.bucket; | |
295 | } | |
296 | } | |
297 | } | |
298 | ||
299 | if (struct_v >= 5) { | |
300 | if (struct_v < 6) { | |
301 | decode(tail_instance, bl); | |
302 | } else { | |
303 | bool need_to_decode; | |
304 | decode(need_to_decode, bl); | |
305 | if (need_to_decode) { | |
306 | decode(tail_instance, bl); | |
307 | } else { | |
308 | tail_instance = obj.key.instance; | |
309 | } | |
310 | } | |
311 | } else { // old object created before 'tail_instance' field added to manifest | |
312 | tail_instance = obj.key.instance; | |
313 | } | |
314 | ||
315 | if (struct_v >= 7) { | |
316 | decode(head_placement_rule, bl); | |
317 | decode(tail_placement.placement_rule, bl); | |
318 | } | |
319 | ||
320 | update_iterators(); | |
321 | DECODE_FINISH(bl); | |
322 | } | |
323 | ||
324 | void dump(Formatter *f) const; | |
325 | static void generate_test_instances(list<RGWObjManifest*>& o); | |
326 | ||
327 | int append(RGWObjManifest& m, const RGWZoneGroup& zonegroup, | |
328 | const RGWZoneParams& zone_params); | |
329 | int append(RGWObjManifest& m, RGWSI_Zone *zone_svc); | |
330 | ||
331 | bool get_rule(uint64_t ofs, RGWObjManifestRule *rule); | |
332 | ||
333 | bool empty() { | |
334 | if (explicit_objs) | |
335 | return objs.empty(); | |
336 | return rules.empty(); | |
337 | } | |
338 | ||
339 | bool has_explicit_objs() { | |
340 | return explicit_objs; | |
341 | } | |
342 | ||
343 | bool has_tail() { | |
344 | if (explicit_objs) { | |
345 | if (objs.size() == 1) { | |
346 | map<uint64_t, RGWObjManifestPart>::iterator iter = objs.begin(); | |
347 | rgw_obj& o = iter->second.loc; | |
348 | return !(obj == o); | |
349 | } | |
350 | return (objs.size() >= 2); | |
351 | } | |
352 | return (obj_size > head_size); | |
353 | } | |
354 | ||
355 | void set_head(const rgw_placement_rule& placement_rule, const rgw_obj& _o, uint64_t _s) { | |
356 | head_placement_rule = placement_rule; | |
357 | obj = _o; | |
358 | head_size = _s; | |
359 | ||
360 | if (explicit_objs && head_size > 0) { | |
361 | objs[0].loc = obj; | |
362 | objs[0].size = head_size; | |
363 | } | |
364 | } | |
365 | ||
366 | const rgw_obj& get_obj() { | |
367 | return obj; | |
368 | } | |
369 | ||
370 | void set_tail_placement(const rgw_placement_rule& placement_rule, const rgw_bucket& _b) { | |
371 | tail_placement.placement_rule = placement_rule; | |
372 | tail_placement.bucket = _b; | |
373 | } | |
374 | ||
375 | const rgw_bucket_placement& get_tail_placement() { | |
376 | return tail_placement; | |
377 | } | |
378 | ||
379 | const rgw_placement_rule& get_head_placement_rule() { | |
380 | return head_placement_rule; | |
381 | } | |
382 | ||
383 | void set_prefix(const string& _p) { | |
384 | prefix = _p; | |
385 | } | |
386 | ||
387 | const string& get_prefix() { | |
388 | return prefix; | |
389 | } | |
390 | ||
391 | void set_tail_instance(const string& _ti) { | |
392 | tail_instance = _ti; | |
393 | } | |
394 | ||
395 | const string& get_tail_instance() { | |
396 | return tail_instance; | |
397 | } | |
398 | ||
399 | void set_head_size(uint64_t _s) { | |
400 | head_size = _s; | |
401 | } | |
402 | ||
403 | void set_obj_size(uint64_t s) { | |
404 | obj_size = s; | |
405 | ||
406 | update_iterators(); | |
407 | } | |
408 | ||
409 | uint64_t get_obj_size() { | |
410 | return obj_size; | |
411 | } | |
412 | ||
413 | uint64_t get_head_size() { | |
414 | return head_size; | |
415 | } | |
416 | ||
417 | uint64_t get_max_head_size() { | |
418 | return max_head_size; | |
419 | } | |
420 | ||
421 | class obj_iterator { | |
422 | RGWObjManifest *manifest = nullptr; | |
423 | uint64_t part_ofs = 0; /* where current part starts */ | |
424 | uint64_t stripe_ofs = 0; /* where current stripe starts */ | |
425 | uint64_t ofs = 0; /* current position within the object */ | |
426 | uint64_t stripe_size = 0; /* current part size */ | |
427 | ||
428 | int cur_part_id = 0; | |
429 | int cur_stripe = 0; | |
430 | string cur_override_prefix; | |
431 | ||
432 | rgw_obj_select location; | |
433 | ||
434 | map<uint64_t, RGWObjManifestRule>::iterator rule_iter; | |
435 | map<uint64_t, RGWObjManifestRule>::iterator next_rule_iter; | |
436 | ||
437 | map<uint64_t, RGWObjManifestPart>::iterator explicit_iter; | |
438 | ||
439 | void update_explicit_pos(); | |
440 | ||
441 | ||
442 | protected: | |
443 | ||
444 | void set_manifest(RGWObjManifest *m) { | |
445 | manifest = m; | |
446 | } | |
447 | ||
448 | public: | |
449 | obj_iterator() = default; | |
450 | explicit obj_iterator(RGWObjManifest *_m) | |
451 | : obj_iterator(_m, 0) | |
452 | {} | |
453 | obj_iterator(RGWObjManifest *_m, uint64_t _ofs) : manifest(_m) { | |
454 | seek(_ofs); | |
455 | } | |
456 | void seek(uint64_t ofs); | |
457 | ||
458 | void operator++(); | |
459 | bool operator==(const obj_iterator& rhs) { | |
460 | return (ofs == rhs.ofs); | |
461 | } | |
462 | bool operator!=(const obj_iterator& rhs) { | |
463 | return (ofs != rhs.ofs); | |
464 | } | |
465 | const rgw_obj_select& get_location() { | |
466 | return location; | |
467 | } | |
468 | ||
469 | /* start of current stripe */ | |
470 | uint64_t get_stripe_ofs() { | |
471 | if (manifest->explicit_objs) { | |
472 | return explicit_iter->first; | |
473 | } | |
474 | return stripe_ofs; | |
475 | } | |
476 | ||
477 | /* current ofs relative to start of rgw object */ | |
478 | uint64_t get_ofs() const { | |
479 | return ofs; | |
480 | } | |
481 | ||
482 | /* stripe number */ | |
483 | int get_cur_stripe() const { | |
484 | return cur_stripe; | |
485 | } | |
486 | ||
487 | /* current stripe size */ | |
488 | uint64_t get_stripe_size() { | |
489 | if (manifest->explicit_objs) { | |
490 | return explicit_iter->second.size; | |
491 | } | |
492 | return stripe_size; | |
493 | } | |
494 | ||
495 | /* offset where data starts within current stripe */ | |
496 | uint64_t location_ofs() { | |
497 | if (manifest->explicit_objs) { | |
498 | return explicit_iter->second.loc_ofs; | |
499 | } | |
500 | return 0; /* all stripes start at zero offset */ | |
501 | } | |
502 | ||
503 | void update_location(); | |
504 | ||
505 | friend class RGWObjManifest; | |
506 | void dump(Formatter *f) const; | |
507 | }; | |
508 | ||
509 | const obj_iterator& obj_begin(); | |
510 | const obj_iterator& obj_end(); | |
511 | obj_iterator obj_find(uint64_t ofs); | |
512 | ||
513 | obj_iterator begin_iter; | |
514 | obj_iterator end_iter; | |
515 | ||
516 | /* | |
517 | * simple object generator. Using a simple single rule manifest. | |
518 | */ | |
519 | class generator { | |
520 | RGWObjManifest *manifest; | |
521 | uint64_t last_ofs; | |
522 | uint64_t cur_part_ofs; | |
523 | int cur_part_id; | |
524 | int cur_stripe; | |
525 | uint64_t cur_stripe_size; | |
526 | string cur_oid; | |
527 | ||
528 | string oid_prefix; | |
529 | ||
530 | rgw_obj_select cur_obj; | |
531 | ||
532 | RGWObjManifestRule rule; | |
533 | ||
534 | public: | |
535 | generator() : manifest(NULL), last_ofs(0), cur_part_ofs(0), cur_part_id(0), | |
536 | cur_stripe(0), cur_stripe_size(0) {} | |
537 | int create_begin(CephContext *cct, RGWObjManifest *manifest, | |
538 | const rgw_placement_rule& head_placement_rule, | |
539 | const rgw_placement_rule *tail_placement_rule, | |
540 | const rgw_bucket& bucket, | |
541 | const rgw_obj& obj); | |
542 | ||
543 | int create_next(uint64_t ofs); | |
544 | ||
545 | rgw_raw_obj get_cur_obj(RGWZoneGroup& zonegroup, RGWZoneParams& zone_params) { return cur_obj.get_raw_obj(zonegroup, zone_params); } | |
546 | rgw_raw_obj get_cur_obj(RGWRados *store) const { return cur_obj.get_raw_obj(store); } | |
547 | ||
548 | /* total max size of current stripe (including head obj) */ | |
549 | uint64_t cur_stripe_max_size() const { | |
550 | return cur_stripe_size; | |
551 | } | |
552 | }; | |
553 | }; | |
554 | WRITE_CLASS_ENCODER(RGWObjManifest) | |
555 |