]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- |
2 | // vim: ts=8 sw=2 smarttab | |
3 | /* | |
4 | * Ceph - scalable distributed file system | |
5 | * | |
6 | * Copyright (C) 2013 eNovance SAS <licensing@enovance.com> | |
7 | * | |
8 | * This is free software; you can redistribute it and/or | |
9 | * modify it under the terms of the GNU Lesser General Public | |
10 | * License version 2.1, as published by the Free Software | |
11 | * Foundation. See file COPYING. | |
12 | * | |
13 | */ | |
14 | #include <iostream> | |
15 | #include "global/global_init.h" | |
16 | #include "common/ceph_argparse.h" | |
17 | #include "rgw/rgw_common.h" | |
18 | #include "rgw/rgw_rados.h" | |
19 | #include "test_rgw_common.h" | |
20 | #define GTEST | |
21 | #ifdef GTEST | |
22 | #include <gtest/gtest.h> | |
23 | #else | |
24 | #define TEST(x, y) void y() | |
25 | #define ASSERT_EQ(v, s) if(v != s)cout << "Error at " << __LINE__ << "(" << #v << "!= " << #s << "\n"; \ | |
26 | else cout << "(" << #v << "==" << #s << ") PASSED\n"; | |
27 | #define EXPECT_EQ(v, s) ASSERT_EQ(v, s) | |
28 | #define ASSERT_TRUE(c) if(c)cout << "Error at " << __LINE__ << "(" << #c << ")" << "\n"; \ | |
29 | else cout << "(" << #c << ") PASSED\n";#define EXPECT_TRUE(c) ASSERT_TRUE(c) | |
30 | #define EXPECT_TRUE(c) ASSERT_TRUE(c) | |
31 | #endif | |
32 | using namespace std; | |
33 | ||
34 | struct OldObjManifestPart { | |
35 | old_rgw_obj loc; /* the object where the data is located */ | |
36 | uint64_t loc_ofs; /* the offset at that object where the data is located */ | |
37 | uint64_t size; /* the part size */ | |
38 | ||
39 | OldObjManifestPart() : loc_ofs(0), size(0) {} | |
40 | ||
41 | void encode(bufferlist& bl) const { | |
42 | ENCODE_START(2, 2, bl); | |
11fdf7f2 TL |
43 | encode(loc, bl); |
44 | encode(loc_ofs, bl); | |
45 | encode(size, bl); | |
7c673cae FG |
46 | ENCODE_FINISH(bl); |
47 | } | |
48 | ||
11fdf7f2 | 49 | void decode(bufferlist::const_iterator& bl) { |
7c673cae | 50 | DECODE_START_LEGACY_COMPAT_LEN_32(2, 2, 2, bl); |
11fdf7f2 TL |
51 | decode(loc, bl); |
52 | decode(loc_ofs, bl); | |
53 | decode(size, bl); | |
7c673cae FG |
54 | DECODE_FINISH(bl); |
55 | } | |
56 | ||
57 | void dump(Formatter *f) const; | |
58 | static void generate_test_instances(list<OldObjManifestPart*>& o); | |
59 | }; | |
60 | WRITE_CLASS_ENCODER(OldObjManifestPart) | |
61 | ||
62 | class OldObjManifest { | |
63 | protected: | |
64 | map<uint64_t, OldObjManifestPart> objs; | |
65 | ||
66 | uint64_t obj_size; | |
67 | public: | |
68 | ||
69 | OldObjManifest() : obj_size(0) {} | |
70 | OldObjManifest(const OldObjManifest& rhs) { | |
71 | *this = rhs; | |
72 | } | |
73 | OldObjManifest& operator=(const OldObjManifest& rhs) { | |
74 | objs = rhs.objs; | |
75 | obj_size = rhs.obj_size; | |
76 | return *this; | |
77 | } | |
78 | ||
79 | const map<uint64_t, OldObjManifestPart>& get_objs() { | |
80 | return objs; | |
81 | } | |
82 | ||
83 | void append(uint64_t ofs, const OldObjManifestPart& part) { | |
84 | objs[ofs] = part; | |
11fdf7f2 | 85 | obj_size = std::max(obj_size, ofs + part.size); |
7c673cae FG |
86 | } |
87 | ||
88 | void encode(bufferlist& bl) const { | |
89 | ENCODE_START(2, 2, bl); | |
11fdf7f2 TL |
90 | encode(obj_size, bl); |
91 | encode(objs, bl); | |
7c673cae FG |
92 | ENCODE_FINISH(bl); |
93 | } | |
94 | ||
11fdf7f2 | 95 | void decode(bufferlist::const_iterator& bl) { |
7c673cae | 96 | DECODE_START_LEGACY_COMPAT_LEN_32(6, 2, 2, bl); |
11fdf7f2 TL |
97 | decode(obj_size, bl); |
98 | decode(objs, bl); | |
7c673cae FG |
99 | DECODE_FINISH(bl); |
100 | } | |
101 | ||
102 | bool empty() { | |
103 | return objs.empty(); | |
104 | } | |
105 | }; | |
106 | WRITE_CLASS_ENCODER(OldObjManifest) | |
107 | ||
108 | void append_head(list<rgw_obj> *objs, rgw_obj& head) | |
109 | { | |
110 | objs->push_back(head); | |
111 | } | |
112 | ||
113 | void append_stripes(list<rgw_obj> *objs, RGWObjManifest& manifest, uint64_t obj_size, uint64_t stripe_size) | |
114 | { | |
115 | string prefix = manifest.get_prefix(); | |
116 | rgw_bucket bucket = manifest.get_obj().bucket; | |
117 | ||
118 | int i = 0; | |
119 | for (uint64_t ofs = manifest.get_max_head_size(); ofs < obj_size; ofs += stripe_size) { | |
120 | char buf[16]; | |
121 | snprintf(buf, sizeof(buf), "%d", ++i); | |
122 | string oid = prefix + buf; | |
123 | cout << "oid=" << oid << std::endl; | |
124 | rgw_obj obj; | |
125 | obj.init_ns(bucket, oid, "shadow"); | |
126 | objs->push_back(obj); | |
127 | } | |
128 | } | |
129 | ||
130 | static void gen_obj(test_rgw_env& env, uint64_t obj_size, uint64_t head_max_size, uint64_t stripe_size, | |
11fdf7f2 | 131 | RGWObjManifest *manifest, const rgw_placement_rule& placement_rule, rgw_bucket *bucket, rgw_obj *head, RGWObjManifest::generator *gen, |
7c673cae FG |
132 | list<rgw_obj> *test_objs) |
133 | { | |
134 | manifest->set_trivial_rule(head_max_size, stripe_size); | |
135 | ||
136 | test_rgw_init_bucket(bucket, "buck"); | |
137 | ||
138 | *head = rgw_obj(*bucket, "oid"); | |
11fdf7f2 | 139 | gen->create_begin(g_ceph_context, manifest, placement_rule, nullptr, *bucket, *head); |
7c673cae FG |
140 | |
141 | append_head(test_objs, *head); | |
142 | cout << "test_objs.size()=" << test_objs->size() << std::endl; | |
143 | append_stripes(test_objs, *manifest, obj_size, stripe_size); | |
144 | ||
145 | cout << "test_objs.size()=" << test_objs->size() << std::endl; | |
146 | ||
147 | ASSERT_EQ((int)manifest->get_obj_size(), 0); | |
148 | ASSERT_EQ((int)manifest->get_head_size(), 0); | |
149 | ASSERT_EQ(manifest->has_tail(), false); | |
150 | ||
151 | uint64_t ofs = 0; | |
152 | list<rgw_obj>::iterator iter = test_objs->begin(); | |
153 | ||
154 | while (ofs < obj_size) { | |
155 | rgw_raw_obj obj = gen->get_cur_obj(env.zonegroup, env.zone_params); | |
156 | cout << "obj=" << obj << std::endl; | |
157 | rgw_raw_obj test_raw = rgw_obj_select(*iter).get_raw_obj(env.zonegroup, env.zone_params); | |
158 | ASSERT_TRUE(obj == test_raw); | |
159 | ||
11fdf7f2 | 160 | ofs = std::min(ofs + gen->cur_stripe_max_size(), obj_size); |
7c673cae FG |
161 | gen->create_next(ofs); |
162 | ||
163 | cout << "obj=" << obj << " *iter=" << *iter << std::endl; | |
164 | cout << "test_objs.size()=" << test_objs->size() << std::endl; | |
165 | ++iter; | |
166 | ||
167 | } | |
168 | ||
169 | if (manifest->has_tail()) { | |
170 | rgw_raw_obj obj = gen->get_cur_obj(env.zonegroup, env.zone_params); | |
171 | rgw_raw_obj test_raw = rgw_obj_select(*iter).get_raw_obj(env.zonegroup, env.zone_params); | |
172 | ASSERT_TRUE(obj == test_raw); | |
173 | ++iter; | |
174 | } | |
175 | ASSERT_TRUE(iter == test_objs->end()); | |
176 | ASSERT_EQ(manifest->get_obj_size(), obj_size); | |
11fdf7f2 | 177 | ASSERT_EQ(manifest->get_head_size(), std::min(obj_size, head_max_size)); |
7c673cae FG |
178 | ASSERT_EQ(manifest->has_tail(), (obj_size > head_max_size)); |
179 | } | |
180 | ||
181 | static void gen_old_obj(test_rgw_env& env, uint64_t obj_size, uint64_t head_max_size, uint64_t stripe_size, | |
182 | OldObjManifest *manifest, old_rgw_bucket *bucket, old_rgw_obj *head, | |
183 | list<old_rgw_obj> *test_objs) | |
184 | { | |
185 | test_rgw_init_old_bucket(bucket, "buck"); | |
186 | ||
187 | *head = old_rgw_obj(*bucket, "obj"); | |
188 | ||
189 | OldObjManifestPart part; | |
190 | part.loc = *head; | |
191 | part.size = head_max_size; | |
192 | part.loc_ofs = 0; | |
193 | ||
194 | manifest->append(0, part); | |
195 | test_objs->push_back(part.loc); | |
196 | ||
197 | string prefix; | |
198 | append_rand_alpha(g_ceph_context, prefix, prefix, 16); | |
199 | ||
200 | int i = 0; | |
201 | for (uint64_t ofs = head_max_size; ofs < obj_size; ofs += stripe_size, i++) { | |
202 | char buf[32]; | |
203 | snprintf(buf, sizeof(buf), "%s.%d", prefix.c_str(), i); | |
204 | old_rgw_obj loc(*bucket, buf); | |
205 | loc.set_ns("shadow"); | |
206 | OldObjManifestPart part; | |
207 | part.loc = loc; | |
208 | part.size = min(stripe_size, obj_size - ofs); | |
209 | part.loc_ofs = 0; | |
210 | ||
211 | manifest->append(ofs, part); | |
212 | ||
213 | test_objs->push_back(loc); | |
214 | } | |
215 | } | |
216 | ||
217 | TEST(TestRGWManifest, head_only_obj) { | |
218 | test_rgw_env env; | |
219 | RGWObjManifest manifest; | |
220 | rgw_bucket bucket; | |
221 | rgw_obj head; | |
222 | RGWObjManifest::generator gen; | |
223 | ||
224 | int obj_size = 256 * 1024; | |
225 | ||
226 | list<rgw_obj> objs; | |
227 | ||
228 | gen_obj(env, obj_size, 512 * 1024, 4 * 1024 * 1024, &manifest, env.zonegroup.default_placement, &bucket, &head, &gen, &objs); | |
229 | ||
230 | cout << " manifest.get_obj_size()=" << manifest.get_obj_size() << std::endl; | |
231 | cout << " manifest.get_head_size()=" << manifest.get_head_size() << std::endl; | |
232 | list<rgw_obj>::iterator liter; | |
233 | ||
234 | RGWObjManifest::obj_iterator iter; | |
235 | for (iter = manifest.obj_begin(), liter = objs.begin(); | |
236 | iter != manifest.obj_end() && liter != objs.end(); | |
237 | ++iter, ++liter) { | |
238 | ASSERT_TRUE(env.get_raw(*liter) == env.get_raw(iter.get_location())); | |
239 | } | |
240 | ||
241 | ASSERT_TRUE(iter == manifest.obj_end()); | |
242 | ASSERT_TRUE(liter == objs.end()); | |
243 | ||
244 | rgw_raw_obj raw_head; | |
245 | ||
246 | iter = manifest.obj_find(100 * 1024); | |
247 | ASSERT_TRUE(env.get_raw(iter.get_location()) == env.get_raw(head)); | |
248 | ASSERT_EQ((int)iter.get_stripe_size(), obj_size); | |
249 | } | |
250 | ||
251 | TEST(TestRGWManifest, obj_with_head_and_tail) { | |
252 | test_rgw_env env; | |
253 | RGWObjManifest manifest; | |
254 | rgw_bucket bucket; | |
255 | rgw_obj head; | |
256 | RGWObjManifest::generator gen; | |
257 | ||
258 | list<rgw_obj> objs; | |
259 | ||
260 | int obj_size = 21 * 1024 * 1024 + 1000; | |
261 | int stripe_size = 4 * 1024 * 1024; | |
262 | int head_size = 512 * 1024; | |
263 | ||
264 | gen_obj(env, obj_size, head_size, stripe_size, &manifest, env.zonegroup.default_placement, &bucket, &head, &gen, &objs); | |
265 | ||
266 | list<rgw_obj>::iterator liter; | |
267 | ||
268 | rgw_obj_select last_obj; | |
269 | ||
270 | RGWObjManifest::obj_iterator iter; | |
271 | for (iter = manifest.obj_begin(), liter = objs.begin(); | |
272 | iter != manifest.obj_end() && liter != objs.end(); | |
273 | ++iter, ++liter) { | |
274 | cout << "*liter=" << *liter << " iter.get_location()=" << env.get_raw(iter.get_location()) << std::endl; | |
275 | ASSERT_TRUE(env.get_raw(*liter) == env.get_raw(iter.get_location())); | |
276 | ||
277 | last_obj = iter.get_location(); | |
278 | } | |
279 | ||
280 | ASSERT_TRUE(iter == manifest.obj_end()); | |
281 | ASSERT_TRUE(liter == objs.end()); | |
282 | ||
283 | iter = manifest.obj_find(100 * 1024); | |
284 | ASSERT_TRUE(env.get_raw(iter.get_location()) == env.get_raw(head)); | |
285 | ASSERT_EQ((int)iter.get_stripe_size(), head_size); | |
286 | ||
287 | uint64_t ofs = 20 * 1024 * 1024 + head_size; | |
288 | iter = manifest.obj_find(ofs + 100); | |
289 | ||
290 | ASSERT_TRUE(env.get_raw(iter.get_location()) == env.get_raw(last_obj)); | |
291 | ASSERT_EQ(iter.get_stripe_ofs(), ofs); | |
292 | ASSERT_EQ(iter.get_stripe_size(), obj_size - ofs); | |
293 | } | |
294 | ||
295 | TEST(TestRGWManifest, multipart) { | |
296 | test_rgw_env env; | |
297 | int num_parts = 16; | |
298 | vector <RGWObjManifest> pm(num_parts); | |
299 | rgw_bucket bucket; | |
300 | uint64_t part_size = 10 * 1024 * 1024; | |
301 | uint64_t stripe_size = 4 * 1024 * 1024; | |
302 | ||
303 | string upload_id = "abc123"; | |
304 | ||
305 | for (int i = 0; i < num_parts; ++i) { | |
306 | RGWObjManifest& manifest = pm[i]; | |
307 | RGWObjManifest::generator gen; | |
308 | manifest.set_prefix(upload_id); | |
309 | ||
310 | manifest.set_multipart_part_rule(stripe_size, i + 1); | |
311 | ||
312 | uint64_t ofs; | |
313 | rgw_obj head; | |
314 | for (ofs = 0; ofs < part_size; ofs += stripe_size) { | |
315 | if (ofs == 0) { | |
11fdf7f2 TL |
316 | rgw_placement_rule rule(env.zonegroup.default_placement.name, RGW_STORAGE_CLASS_STANDARD); |
317 | int r = gen.create_begin(g_ceph_context, &manifest, rule, nullptr, bucket, head); | |
7c673cae FG |
318 | ASSERT_EQ(r, 0); |
319 | continue; | |
320 | } | |
321 | gen.create_next(ofs); | |
322 | } | |
323 | ||
324 | if (ofs > part_size) { | |
325 | gen.create_next(part_size); | |
326 | } | |
327 | } | |
328 | ||
329 | RGWObjManifest m; | |
330 | ||
331 | for (int i = 0; i < num_parts; i++) { | |
332 | m.append(pm[i], env.zonegroup, env.zone_params); | |
333 | } | |
334 | RGWObjManifest::obj_iterator iter; | |
335 | for (iter = m.obj_begin(); iter != m.obj_end(); ++iter) { | |
336 | RGWObjManifest::obj_iterator fiter = m.obj_find(iter.get_ofs()); | |
337 | ASSERT_TRUE(env.get_raw(fiter.get_location()) == env.get_raw(iter.get_location())); | |
338 | } | |
339 | ||
340 | ASSERT_EQ(m.get_obj_size(), num_parts * part_size); | |
341 | } | |
342 | ||
343 | TEST(TestRGWManifest, old_obj_manifest) { | |
344 | test_rgw_env env; | |
345 | OldObjManifest old_manifest; | |
346 | old_rgw_bucket old_bucket; | |
347 | old_rgw_obj old_head; | |
348 | ||
349 | int obj_size = 40 * 1024 * 1024; | |
350 | uint64_t stripe_size = 4 * 1024 * 1024; | |
351 | uint64_t head_size = 512 * 1024; | |
352 | ||
353 | list<old_rgw_obj> old_objs; | |
354 | ||
355 | gen_old_obj(env, obj_size, head_size, stripe_size, &old_manifest, &old_bucket, &old_head, &old_objs); | |
356 | ||
357 | ASSERT_EQ(old_objs.size(), 11u); | |
358 | ||
359 | ||
360 | bufferlist bl; | |
11fdf7f2 | 361 | encode(old_manifest , bl); |
7c673cae FG |
362 | |
363 | RGWObjManifest manifest; | |
364 | ||
365 | try { | |
11fdf7f2 TL |
366 | auto iter = bl.cbegin(); |
367 | decode(manifest, iter); | |
7c673cae FG |
368 | } catch (buffer::error& err) { |
369 | ASSERT_TRUE(false); | |
370 | } | |
371 | ||
372 | rgw_raw_obj last_obj; | |
373 | ||
374 | RGWObjManifest::obj_iterator iter; | |
375 | auto liter = old_objs.begin(); | |
376 | for (iter = manifest.obj_begin(); | |
377 | iter != manifest.obj_end() && liter != old_objs.end(); | |
378 | ++iter, ++liter) { | |
379 | rgw_pool old_pool(liter->bucket.data_pool); | |
380 | string old_oid; | |
381 | prepend_old_bucket_marker(old_bucket, liter->get_object(), old_oid); | |
382 | rgw_raw_obj raw_old(old_pool, old_oid); | |
383 | cout << "*liter=" << raw_old << " iter.get_location()=" << env.get_raw(iter.get_location()) << std::endl; | |
384 | ASSERT_EQ(raw_old, env.get_raw(iter.get_location())); | |
385 | ||
386 | last_obj = env.get_raw(iter.get_location()); | |
387 | } | |
388 | ||
389 | ASSERT_TRUE(liter == old_objs.end()); | |
390 | ASSERT_TRUE(iter == manifest.obj_end()); | |
391 | ||
392 | } | |
393 | ||
394 | ||
395 | int main(int argc, char **argv) { | |
396 | vector<const char*> args; | |
397 | argv_to_vec(argc, (const char **)argv, args); | |
7c673cae | 398 | |
11fdf7f2 TL |
399 | auto cct = global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, |
400 | CODE_ENVIRONMENT_UTILITY, | |
401 | CINIT_FLAG_NO_DEFAULT_CONFIG_FILE); | |
7c673cae FG |
402 | common_init_finish(g_ceph_context); |
403 | ::testing::InitGoogleTest(&argc, argv); | |
404 | return RUN_ALL_TESTS(); | |
405 | } | |
406 |