]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- |
2 | // vim: ts=8 sw=2 smarttab | |
3 | /* | |
4 | * Ceph - scalable distributed file system | |
5 | * | |
6 | * Copyright (C) 2013 eNovance SAS <licensing@enovance.com> | |
7 | * | |
8 | * This is free software; you can redistribute it and/or | |
9 | * modify it under the terms of the GNU Lesser General Public | |
10 | * License version 2.1, as published by the Free Software | |
11 | * Foundation. See file COPYING. | |
12 | * | |
13 | */ | |
14 | #include <iostream> | |
15 | #include "global/global_init.h" | |
16 | #include "common/ceph_argparse.h" | |
1e59de90 TL |
17 | #include "rgw_common.h" |
18 | #include "rgw_rados.h" | |
7c673cae | 19 | #include "test_rgw_common.h" |
7c673cae | 20 | #include <gtest/gtest.h> |
9f95a23c | 21 | |
7c673cae FG |
22 | using namespace std; |
23 | ||
b3b6e05e TL |
24 | auto cct = new CephContext(CEPH_ENTITY_TYPE_CLIENT); |
25 | const DoutPrefix dp(cct, 1, "test rgw manifest: "); | |
26 | ||
7c673cae FG |
27 | struct OldObjManifestPart { |
28 | old_rgw_obj loc; /* the object where the data is located */ | |
29 | uint64_t loc_ofs; /* the offset at that object where the data is located */ | |
30 | uint64_t size; /* the part size */ | |
31 | ||
32 | OldObjManifestPart() : loc_ofs(0), size(0) {} | |
33 | ||
34 | void encode(bufferlist& bl) const { | |
35 | ENCODE_START(2, 2, bl); | |
11fdf7f2 TL |
36 | encode(loc, bl); |
37 | encode(loc_ofs, bl); | |
38 | encode(size, bl); | |
7c673cae FG |
39 | ENCODE_FINISH(bl); |
40 | } | |
41 | ||
11fdf7f2 | 42 | void decode(bufferlist::const_iterator& bl) { |
7c673cae | 43 | DECODE_START_LEGACY_COMPAT_LEN_32(2, 2, 2, bl); |
11fdf7f2 TL |
44 | decode(loc, bl); |
45 | decode(loc_ofs, bl); | |
46 | decode(size, bl); | |
7c673cae FG |
47 | DECODE_FINISH(bl); |
48 | } | |
49 | ||
50 | void dump(Formatter *f) const; | |
51 | static void generate_test_instances(list<OldObjManifestPart*>& o); | |
52 | }; | |
53 | WRITE_CLASS_ENCODER(OldObjManifestPart) | |
54 | ||
55 | class OldObjManifest { | |
56 | protected: | |
57 | map<uint64_t, OldObjManifestPart> objs; | |
58 | ||
59 | uint64_t obj_size; | |
60 | public: | |
61 | ||
62 | OldObjManifest() : obj_size(0) {} | |
63 | OldObjManifest(const OldObjManifest& rhs) { | |
64 | *this = rhs; | |
65 | } | |
66 | OldObjManifest& operator=(const OldObjManifest& rhs) { | |
67 | objs = rhs.objs; | |
68 | obj_size = rhs.obj_size; | |
69 | return *this; | |
70 | } | |
71 | ||
72 | const map<uint64_t, OldObjManifestPart>& get_objs() { | |
73 | return objs; | |
74 | } | |
75 | ||
76 | void append(uint64_t ofs, const OldObjManifestPart& part) { | |
77 | objs[ofs] = part; | |
11fdf7f2 | 78 | obj_size = std::max(obj_size, ofs + part.size); |
7c673cae FG |
79 | } |
80 | ||
81 | void encode(bufferlist& bl) const { | |
82 | ENCODE_START(2, 2, bl); | |
11fdf7f2 TL |
83 | encode(obj_size, bl); |
84 | encode(objs, bl); | |
7c673cae FG |
85 | ENCODE_FINISH(bl); |
86 | } | |
87 | ||
11fdf7f2 | 88 | void decode(bufferlist::const_iterator& bl) { |
7c673cae | 89 | DECODE_START_LEGACY_COMPAT_LEN_32(6, 2, 2, bl); |
11fdf7f2 TL |
90 | decode(obj_size, bl); |
91 | decode(objs, bl); | |
7c673cae FG |
92 | DECODE_FINISH(bl); |
93 | } | |
94 | ||
95 | bool empty() { | |
96 | return objs.empty(); | |
97 | } | |
98 | }; | |
99 | WRITE_CLASS_ENCODER(OldObjManifest) | |
100 | ||
101 | void append_head(list<rgw_obj> *objs, rgw_obj& head) | |
102 | { | |
103 | objs->push_back(head); | |
104 | } | |
105 | ||
106 | void append_stripes(list<rgw_obj> *objs, RGWObjManifest& manifest, uint64_t obj_size, uint64_t stripe_size) | |
107 | { | |
108 | string prefix = manifest.get_prefix(); | |
109 | rgw_bucket bucket = manifest.get_obj().bucket; | |
110 | ||
111 | int i = 0; | |
112 | for (uint64_t ofs = manifest.get_max_head_size(); ofs < obj_size; ofs += stripe_size) { | |
113 | char buf[16]; | |
114 | snprintf(buf, sizeof(buf), "%d", ++i); | |
115 | string oid = prefix + buf; | |
116 | cout << "oid=" << oid << std::endl; | |
117 | rgw_obj obj; | |
118 | obj.init_ns(bucket, oid, "shadow"); | |
119 | objs->push_back(obj); | |
120 | } | |
121 | } | |
122 | ||
123 | static void gen_obj(test_rgw_env& env, uint64_t obj_size, uint64_t head_max_size, uint64_t stripe_size, | |
11fdf7f2 | 124 | RGWObjManifest *manifest, const rgw_placement_rule& placement_rule, rgw_bucket *bucket, rgw_obj *head, RGWObjManifest::generator *gen, |
7c673cae FG |
125 | list<rgw_obj> *test_objs) |
126 | { | |
127 | manifest->set_trivial_rule(head_max_size, stripe_size); | |
128 | ||
129 | test_rgw_init_bucket(bucket, "buck"); | |
130 | ||
131 | *head = rgw_obj(*bucket, "oid"); | |
11fdf7f2 | 132 | gen->create_begin(g_ceph_context, manifest, placement_rule, nullptr, *bucket, *head); |
7c673cae FG |
133 | |
134 | append_head(test_objs, *head); | |
135 | cout << "test_objs.size()=" << test_objs->size() << std::endl; | |
136 | append_stripes(test_objs, *manifest, obj_size, stripe_size); | |
137 | ||
138 | cout << "test_objs.size()=" << test_objs->size() << std::endl; | |
139 | ||
140 | ASSERT_EQ((int)manifest->get_obj_size(), 0); | |
141 | ASSERT_EQ((int)manifest->get_head_size(), 0); | |
142 | ASSERT_EQ(manifest->has_tail(), false); | |
143 | ||
144 | uint64_t ofs = 0; | |
145 | list<rgw_obj>::iterator iter = test_objs->begin(); | |
146 | ||
147 | while (ofs < obj_size) { | |
148 | rgw_raw_obj obj = gen->get_cur_obj(env.zonegroup, env.zone_params); | |
149 | cout << "obj=" << obj << std::endl; | |
150 | rgw_raw_obj test_raw = rgw_obj_select(*iter).get_raw_obj(env.zonegroup, env.zone_params); | |
151 | ASSERT_TRUE(obj == test_raw); | |
152 | ||
11fdf7f2 | 153 | ofs = std::min(ofs + gen->cur_stripe_max_size(), obj_size); |
7c673cae FG |
154 | gen->create_next(ofs); |
155 | ||
156 | cout << "obj=" << obj << " *iter=" << *iter << std::endl; | |
157 | cout << "test_objs.size()=" << test_objs->size() << std::endl; | |
158 | ++iter; | |
159 | ||
160 | } | |
161 | ||
162 | if (manifest->has_tail()) { | |
163 | rgw_raw_obj obj = gen->get_cur_obj(env.zonegroup, env.zone_params); | |
164 | rgw_raw_obj test_raw = rgw_obj_select(*iter).get_raw_obj(env.zonegroup, env.zone_params); | |
165 | ASSERT_TRUE(obj == test_raw); | |
166 | ++iter; | |
167 | } | |
168 | ASSERT_TRUE(iter == test_objs->end()); | |
169 | ASSERT_EQ(manifest->get_obj_size(), obj_size); | |
11fdf7f2 | 170 | ASSERT_EQ(manifest->get_head_size(), std::min(obj_size, head_max_size)); |
7c673cae FG |
171 | ASSERT_EQ(manifest->has_tail(), (obj_size > head_max_size)); |
172 | } | |
173 | ||
174 | static void gen_old_obj(test_rgw_env& env, uint64_t obj_size, uint64_t head_max_size, uint64_t stripe_size, | |
175 | OldObjManifest *manifest, old_rgw_bucket *bucket, old_rgw_obj *head, | |
176 | list<old_rgw_obj> *test_objs) | |
177 | { | |
178 | test_rgw_init_old_bucket(bucket, "buck"); | |
179 | ||
180 | *head = old_rgw_obj(*bucket, "obj"); | |
181 | ||
182 | OldObjManifestPart part; | |
183 | part.loc = *head; | |
184 | part.size = head_max_size; | |
185 | part.loc_ofs = 0; | |
186 | ||
187 | manifest->append(0, part); | |
188 | test_objs->push_back(part.loc); | |
189 | ||
190 | string prefix; | |
191 | append_rand_alpha(g_ceph_context, prefix, prefix, 16); | |
192 | ||
193 | int i = 0; | |
194 | for (uint64_t ofs = head_max_size; ofs < obj_size; ofs += stripe_size, i++) { | |
195 | char buf[32]; | |
196 | snprintf(buf, sizeof(buf), "%s.%d", prefix.c_str(), i); | |
197 | old_rgw_obj loc(*bucket, buf); | |
198 | loc.set_ns("shadow"); | |
199 | OldObjManifestPart part; | |
200 | part.loc = loc; | |
201 | part.size = min(stripe_size, obj_size - ofs); | |
202 | part.loc_ofs = 0; | |
203 | ||
204 | manifest->append(ofs, part); | |
205 | ||
206 | test_objs->push_back(loc); | |
207 | } | |
208 | } | |
209 | ||
210 | TEST(TestRGWManifest, head_only_obj) { | |
211 | test_rgw_env env; | |
212 | RGWObjManifest manifest; | |
213 | rgw_bucket bucket; | |
214 | rgw_obj head; | |
215 | RGWObjManifest::generator gen; | |
216 | ||
217 | int obj_size = 256 * 1024; | |
218 | ||
219 | list<rgw_obj> objs; | |
220 | ||
221 | gen_obj(env, obj_size, 512 * 1024, 4 * 1024 * 1024, &manifest, env.zonegroup.default_placement, &bucket, &head, &gen, &objs); | |
222 | ||
223 | cout << " manifest.get_obj_size()=" << manifest.get_obj_size() << std::endl; | |
224 | cout << " manifest.get_head_size()=" << manifest.get_head_size() << std::endl; | |
225 | list<rgw_obj>::iterator liter; | |
226 | ||
227 | RGWObjManifest::obj_iterator iter; | |
b3b6e05e TL |
228 | for (iter = manifest.obj_begin(&dp), liter = objs.begin(); |
229 | iter != manifest.obj_end(&dp) && liter != objs.end(); | |
7c673cae FG |
230 | ++iter, ++liter) { |
231 | ASSERT_TRUE(env.get_raw(*liter) == env.get_raw(iter.get_location())); | |
232 | } | |
233 | ||
b3b6e05e | 234 | ASSERT_TRUE(iter == manifest.obj_end(&dp)); |
7c673cae FG |
235 | ASSERT_TRUE(liter == objs.end()); |
236 | ||
237 | rgw_raw_obj raw_head; | |
238 | ||
b3b6e05e | 239 | iter = manifest.obj_find(&dp, 100 * 1024); |
7c673cae FG |
240 | ASSERT_TRUE(env.get_raw(iter.get_location()) == env.get_raw(head)); |
241 | ASSERT_EQ((int)iter.get_stripe_size(), obj_size); | |
242 | } | |
243 | ||
244 | TEST(TestRGWManifest, obj_with_head_and_tail) { | |
245 | test_rgw_env env; | |
246 | RGWObjManifest manifest; | |
247 | rgw_bucket bucket; | |
248 | rgw_obj head; | |
249 | RGWObjManifest::generator gen; | |
250 | ||
251 | list<rgw_obj> objs; | |
252 | ||
253 | int obj_size = 21 * 1024 * 1024 + 1000; | |
254 | int stripe_size = 4 * 1024 * 1024; | |
255 | int head_size = 512 * 1024; | |
256 | ||
257 | gen_obj(env, obj_size, head_size, stripe_size, &manifest, env.zonegroup.default_placement, &bucket, &head, &gen, &objs); | |
258 | ||
259 | list<rgw_obj>::iterator liter; | |
260 | ||
261 | rgw_obj_select last_obj; | |
262 | ||
263 | RGWObjManifest::obj_iterator iter; | |
b3b6e05e TL |
264 | for (iter = manifest.obj_begin(&dp), liter = objs.begin(); |
265 | iter != manifest.obj_end(&dp) && liter != objs.end(); | |
7c673cae FG |
266 | ++iter, ++liter) { |
267 | cout << "*liter=" << *liter << " iter.get_location()=" << env.get_raw(iter.get_location()) << std::endl; | |
268 | ASSERT_TRUE(env.get_raw(*liter) == env.get_raw(iter.get_location())); | |
269 | ||
270 | last_obj = iter.get_location(); | |
271 | } | |
272 | ||
b3b6e05e | 273 | ASSERT_TRUE(iter == manifest.obj_end(&dp)); |
7c673cae FG |
274 | ASSERT_TRUE(liter == objs.end()); |
275 | ||
b3b6e05e | 276 | iter = manifest.obj_find(&dp, 100 * 1024); |
7c673cae FG |
277 | ASSERT_TRUE(env.get_raw(iter.get_location()) == env.get_raw(head)); |
278 | ASSERT_EQ((int)iter.get_stripe_size(), head_size); | |
279 | ||
280 | uint64_t ofs = 20 * 1024 * 1024 + head_size; | |
b3b6e05e | 281 | iter = manifest.obj_find(&dp, ofs + 100); |
7c673cae FG |
282 | |
283 | ASSERT_TRUE(env.get_raw(iter.get_location()) == env.get_raw(last_obj)); | |
284 | ASSERT_EQ(iter.get_stripe_ofs(), ofs); | |
285 | ASSERT_EQ(iter.get_stripe_size(), obj_size - ofs); | |
286 | } | |
287 | ||
288 | TEST(TestRGWManifest, multipart) { | |
289 | test_rgw_env env; | |
290 | int num_parts = 16; | |
291 | vector <RGWObjManifest> pm(num_parts); | |
292 | rgw_bucket bucket; | |
293 | uint64_t part_size = 10 * 1024 * 1024; | |
294 | uint64_t stripe_size = 4 * 1024 * 1024; | |
295 | ||
296 | string upload_id = "abc123"; | |
297 | ||
298 | for (int i = 0; i < num_parts; ++i) { | |
299 | RGWObjManifest& manifest = pm[i]; | |
300 | RGWObjManifest::generator gen; | |
301 | manifest.set_prefix(upload_id); | |
302 | ||
303 | manifest.set_multipart_part_rule(stripe_size, i + 1); | |
304 | ||
305 | uint64_t ofs; | |
306 | rgw_obj head; | |
307 | for (ofs = 0; ofs < part_size; ofs += stripe_size) { | |
308 | if (ofs == 0) { | |
11fdf7f2 TL |
309 | rgw_placement_rule rule(env.zonegroup.default_placement.name, RGW_STORAGE_CLASS_STANDARD); |
310 | int r = gen.create_begin(g_ceph_context, &manifest, rule, nullptr, bucket, head); | |
7c673cae FG |
311 | ASSERT_EQ(r, 0); |
312 | continue; | |
313 | } | |
314 | gen.create_next(ofs); | |
315 | } | |
316 | ||
317 | if (ofs > part_size) { | |
318 | gen.create_next(part_size); | |
319 | } | |
320 | } | |
321 | ||
322 | RGWObjManifest m; | |
323 | ||
324 | for (int i = 0; i < num_parts; i++) { | |
b3b6e05e | 325 | m.append(&dp, pm[i], env.zonegroup, env.zone_params); |
7c673cae FG |
326 | } |
327 | RGWObjManifest::obj_iterator iter; | |
b3b6e05e TL |
328 | for (iter = m.obj_begin(&dp); iter != m.obj_end(&dp); ++iter) { |
329 | RGWObjManifest::obj_iterator fiter = m.obj_find(&dp, iter.get_ofs()); | |
7c673cae FG |
330 | ASSERT_TRUE(env.get_raw(fiter.get_location()) == env.get_raw(iter.get_location())); |
331 | } | |
332 | ||
333 | ASSERT_EQ(m.get_obj_size(), num_parts * part_size); | |
334 | } | |
335 | ||
336 | TEST(TestRGWManifest, old_obj_manifest) { | |
337 | test_rgw_env env; | |
338 | OldObjManifest old_manifest; | |
339 | old_rgw_bucket old_bucket; | |
340 | old_rgw_obj old_head; | |
341 | ||
342 | int obj_size = 40 * 1024 * 1024; | |
343 | uint64_t stripe_size = 4 * 1024 * 1024; | |
344 | uint64_t head_size = 512 * 1024; | |
345 | ||
346 | list<old_rgw_obj> old_objs; | |
347 | ||
348 | gen_old_obj(env, obj_size, head_size, stripe_size, &old_manifest, &old_bucket, &old_head, &old_objs); | |
349 | ||
350 | ASSERT_EQ(old_objs.size(), 11u); | |
351 | ||
352 | ||
353 | bufferlist bl; | |
11fdf7f2 | 354 | encode(old_manifest , bl); |
7c673cae FG |
355 | |
356 | RGWObjManifest manifest; | |
357 | ||
358 | try { | |
11fdf7f2 TL |
359 | auto iter = bl.cbegin(); |
360 | decode(manifest, iter); | |
7c673cae FG |
361 | } catch (buffer::error& err) { |
362 | ASSERT_TRUE(false); | |
363 | } | |
364 | ||
365 | rgw_raw_obj last_obj; | |
366 | ||
367 | RGWObjManifest::obj_iterator iter; | |
368 | auto liter = old_objs.begin(); | |
b3b6e05e TL |
369 | for (iter = manifest.obj_begin(&dp); |
370 | iter != manifest.obj_end(&dp) && liter != old_objs.end(); | |
7c673cae FG |
371 | ++iter, ++liter) { |
372 | rgw_pool old_pool(liter->bucket.data_pool); | |
373 | string old_oid; | |
374 | prepend_old_bucket_marker(old_bucket, liter->get_object(), old_oid); | |
375 | rgw_raw_obj raw_old(old_pool, old_oid); | |
376 | cout << "*liter=" << raw_old << " iter.get_location()=" << env.get_raw(iter.get_location()) << std::endl; | |
377 | ASSERT_EQ(raw_old, env.get_raw(iter.get_location())); | |
378 | ||
379 | last_obj = env.get_raw(iter.get_location()); | |
380 | } | |
381 | ||
382 | ASSERT_TRUE(liter == old_objs.end()); | |
b3b6e05e | 383 | ASSERT_TRUE(iter == manifest.obj_end(&dp)); |
7c673cae FG |
384 | |
385 | } | |
386 | ||
387 | ||
388 | int main(int argc, char **argv) { | |
20effc67 | 389 | auto args = argv_to_vec(argc, argv); |
11fdf7f2 TL |
390 | auto cct = global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, |
391 | CODE_ENVIRONMENT_UTILITY, | |
392 | CINIT_FLAG_NO_DEFAULT_CONFIG_FILE); | |
7c673cae FG |
393 | common_init_finish(g_ceph_context); |
394 | ::testing::InitGoogleTest(&argc, argv); | |
395 | return RUN_ALL_TESTS(); | |
396 | } | |
397 |