]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- |
2 | #include "include/memory.h" | |
3 | #include <map> | |
4 | #include <set> | |
5 | #include <boost/scoped_ptr.hpp> | |
6 | ||
7 | #include "include/buffer.h" | |
8 | #include "test/ObjectMap/KeyValueDBMemory.h" | |
9 | #include "kv/KeyValueDB.h" | |
10 | #include "os/filestore/DBObjectMap.h" | |
11 | #include "os/filestore/HashIndex.h" | |
12 | #include <sys/types.h> | |
13 | #include "global/global_init.h" | |
14 | #include "common/ceph_argparse.h" | |
15 | #include <dirent.h> | |
16 | ||
17 | #include "gtest/gtest.h" | |
18 | #include "stdlib.h" | |
19 | ||
20 | using namespace std; | |
21 | ||
22 | template <typename T> | |
23 | typename T::iterator rand_choose(T &cont) { | |
24 | if (cont.size() == 0) { | |
25 | return cont.end(); | |
26 | } | |
27 | int index = rand() % cont.size(); | |
28 | typename T::iterator retval = cont.begin(); | |
29 | ||
30 | for (; index > 0; --index) ++retval; | |
31 | return retval; | |
32 | } | |
33 | ||
34 | string num_str(unsigned i) { | |
35 | char buf[100]; | |
36 | snprintf(buf, sizeof(buf), "%.10u", i); | |
37 | return string(buf); | |
38 | } | |
39 | ||
40 | class ObjectMapTester { | |
41 | public: | |
42 | ObjectMap *db; | |
43 | set<string> key_space; | |
44 | set<string> object_name_space; | |
45 | map<string, map<string, string> > omap; | |
46 | map<string, string > hmap; | |
47 | map<string, map<string, string> > xattrs; | |
48 | unsigned seq; | |
49 | ||
50 | ObjectMapTester() : db(0), seq(0) {} | |
51 | ||
52 | string val_from_key(const string &object, const string &key) { | |
53 | return object + "_" + key + "_" + num_str(seq++); | |
54 | } | |
55 | ||
56 | void set_key(const string &objname, const string &key, const string &value) { | |
57 | set_key(ghobject_t(hobject_t(sobject_t(objname, CEPH_NOSNAP))), | |
58 | key, value); | |
59 | } | |
60 | ||
61 | void set_xattr(const string &objname, const string &key, const string &value) { | |
62 | set_xattr(ghobject_t(hobject_t(sobject_t(objname, CEPH_NOSNAP))), | |
63 | key, value); | |
64 | } | |
65 | ||
66 | void set_key(ghobject_t hoid, | |
67 | string key, string value) { | |
68 | map<string, bufferlist> to_write; | |
69 | bufferptr bp(value.c_str(), value.size()); | |
70 | bufferlist bl; | |
71 | bl.append(bp); | |
72 | to_write.insert(make_pair(key, bl)); | |
73 | db->set_keys(hoid, to_write); | |
74 | } | |
75 | ||
76 | void set_keys(ghobject_t hoid, const map<string, string> &to_set) { | |
77 | map<string, bufferlist> to_write; | |
78 | for (auto &&i: to_set) { | |
79 | bufferptr bp(i.second.data(), i.second.size()); | |
80 | bufferlist bl; | |
81 | bl.append(bp); | |
82 | to_write.insert(make_pair(i.first, bl)); | |
83 | } | |
84 | db->set_keys(hoid, to_write); | |
85 | } | |
86 | ||
87 | void set_xattr(ghobject_t hoid, | |
88 | string key, string value) { | |
89 | map<string, bufferlist> to_write; | |
90 | bufferptr bp(value.c_str(), value.size()); | |
91 | bufferlist bl; | |
92 | bl.append(bp); | |
93 | to_write.insert(make_pair(key, bl)); | |
94 | db->set_xattrs(hoid, to_write); | |
95 | } | |
96 | ||
97 | void set_header(const string &objname, const string &value) { | |
98 | set_header(ghobject_t(hobject_t(sobject_t(objname, CEPH_NOSNAP))), | |
99 | value); | |
100 | } | |
101 | ||
102 | void set_header(ghobject_t hoid, | |
103 | const string &value) { | |
104 | bufferlist header; | |
105 | header.append(bufferptr(value.c_str(), value.size() + 1)); | |
106 | db->set_header(hoid, header); | |
107 | } | |
108 | ||
109 | int get_header(const string &objname, string *value) { | |
110 | return get_header(ghobject_t(hobject_t(sobject_t(objname, CEPH_NOSNAP))), | |
111 | value); | |
112 | } | |
113 | ||
114 | int get_header(ghobject_t hoid, | |
115 | string *value) { | |
116 | bufferlist header; | |
117 | int r = db->get_header(hoid, &header); | |
118 | if (r < 0) | |
119 | return r; | |
120 | if (header.length()) | |
121 | *value = string(header.c_str()); | |
122 | else | |
123 | *value = string(""); | |
124 | return 0; | |
125 | } | |
126 | ||
127 | int get_xattr(const string &objname, const string &key, string *value) { | |
128 | return get_xattr(ghobject_t(hobject_t(sobject_t(objname, CEPH_NOSNAP))), | |
129 | key, value); | |
130 | } | |
131 | ||
132 | int get_xattr(ghobject_t hoid, | |
133 | string key, string *value) { | |
134 | set<string> to_get; | |
135 | to_get.insert(key); | |
136 | map<string, bufferlist> got; | |
137 | db->get_xattrs(hoid, to_get, &got); | |
138 | if (!got.empty()) { | |
139 | *value = string(got.begin()->second.c_str(), | |
140 | got.begin()->second.length()); | |
141 | return 1; | |
142 | } else { | |
143 | return 0; | |
144 | } | |
145 | } | |
146 | ||
147 | int get_key(const string &objname, const string &key, string *value) { | |
148 | return get_key(ghobject_t(hobject_t(sobject_t(objname, CEPH_NOSNAP))), | |
149 | key, value); | |
150 | } | |
151 | ||
152 | int get_key(ghobject_t hoid, | |
153 | string key, string *value) { | |
154 | set<string> to_get; | |
155 | to_get.insert(key); | |
156 | map<string, bufferlist> got; | |
157 | db->get_values(hoid, to_get, &got); | |
158 | if (!got.empty()) { | |
159 | if (value) { | |
160 | *value = string(got.begin()->second.c_str(), | |
161 | got.begin()->second.length()); | |
162 | } | |
163 | return 1; | |
164 | } else { | |
165 | return 0; | |
166 | } | |
167 | } | |
168 | ||
169 | void remove_key(const string &objname, const string &key) { | |
170 | remove_key(ghobject_t(hobject_t(sobject_t(objname, CEPH_NOSNAP))), | |
171 | key); | |
172 | } | |
173 | ||
174 | void remove_keys(const string &objname, const set<string> &to_remove) { | |
175 | remove_keys(ghobject_t(hobject_t(sobject_t(objname, CEPH_NOSNAP))), | |
176 | to_remove); | |
177 | } | |
178 | ||
179 | void remove_key(ghobject_t hoid, | |
180 | string key) { | |
181 | set<string> to_remove; | |
182 | to_remove.insert(key); | |
183 | db->rm_keys(hoid, to_remove); | |
184 | } | |
185 | ||
186 | void remove_keys(ghobject_t hoid, | |
187 | const set<string> &to_remove) { | |
188 | db->rm_keys(hoid, to_remove); | |
189 | } | |
190 | ||
191 | void remove_xattr(const string &objname, const string &key) { | |
192 | remove_xattr(ghobject_t(hobject_t(sobject_t(objname, CEPH_NOSNAP))), | |
193 | key); | |
194 | } | |
195 | ||
196 | void remove_xattr(ghobject_t hoid, | |
197 | string key) { | |
198 | set<string> to_remove; | |
199 | to_remove.insert(key); | |
200 | db->remove_xattrs(hoid, to_remove); | |
201 | } | |
202 | ||
203 | void clone(const string &objname, const string &target) { | |
204 | clone(ghobject_t(hobject_t(sobject_t(objname, CEPH_NOSNAP))), | |
205 | ghobject_t(hobject_t(sobject_t(target, CEPH_NOSNAP)))); | |
206 | } | |
207 | ||
208 | void clone(ghobject_t hoid, | |
209 | ghobject_t hoid2) { | |
210 | db->clone(hoid, hoid2); | |
211 | } | |
212 | ||
213 | void rename(const string &objname, const string &target) { | |
214 | rename(ghobject_t(hobject_t(sobject_t(objname, CEPH_NOSNAP))), | |
215 | ghobject_t(hobject_t(sobject_t(target, CEPH_NOSNAP)))); | |
216 | } | |
217 | ||
218 | void rename(ghobject_t hoid, | |
219 | ghobject_t hoid2) { | |
220 | db->rename(hoid, hoid2); | |
221 | } | |
222 | ||
223 | void clear(const string &objname) { | |
224 | clear(ghobject_t(hobject_t(sobject_t(objname, CEPH_NOSNAP)))); | |
225 | } | |
226 | ||
227 | void legacy_clone(const string &objname, const string &target) { | |
228 | legacy_clone(ghobject_t(hobject_t(sobject_t(objname, CEPH_NOSNAP))), | |
229 | ghobject_t(hobject_t(sobject_t(target, CEPH_NOSNAP)))); | |
230 | } | |
231 | ||
232 | void legacy_clone(ghobject_t hoid, | |
233 | ghobject_t hoid2) { | |
234 | db->legacy_clone(hoid, hoid2); | |
235 | } | |
236 | ||
237 | void clear(ghobject_t hoid) { | |
238 | db->clear(hoid); | |
239 | } | |
240 | ||
241 | void clear_omap(const string &objname) { | |
242 | clear_omap(ghobject_t(hobject_t(sobject_t(objname, CEPH_NOSNAP)))); | |
243 | } | |
244 | ||
245 | void clear_omap(const ghobject_t &objname) { | |
246 | db->clear_keys_header(objname); | |
247 | } | |
248 | ||
249 | void def_init() { | |
250 | for (unsigned i = 0; i < 10000; ++i) { | |
251 | key_space.insert("key_" + num_str(i)); | |
252 | } | |
253 | for (unsigned i = 0; i < 100; ++i) { | |
254 | object_name_space.insert("name_" + num_str(i)); | |
255 | } | |
256 | } | |
257 | ||
258 | void init_key_set(const set<string> &keys) { | |
259 | key_space = keys; | |
260 | } | |
261 | ||
262 | void init_object_name_space(const set<string> &onamespace) { | |
263 | object_name_space = onamespace; | |
264 | } | |
265 | ||
266 | void auto_set_xattr(ostream &out) { | |
267 | set<string>::iterator key = rand_choose(key_space); | |
268 | set<string>::iterator object = rand_choose(object_name_space); | |
269 | ||
270 | string value = val_from_key(*object, *key); | |
271 | ||
272 | xattrs[*object][*key] = value; | |
273 | set_xattr(*object, *key, value); | |
274 | ||
275 | out << "auto_set_xattr " << *object << ": " << *key << " -> " | |
276 | << value << std::endl; | |
277 | } | |
278 | ||
279 | void test_set_key(const string &obj, const string &key, const string &val) { | |
280 | omap[obj][key] = val; | |
281 | set_key(obj, key, val); | |
282 | } | |
283 | ||
284 | void test_set_keys(const string &obj, const map<string, string> &to_set) { | |
285 | for (auto &&i: to_set) { | |
286 | omap[obj][i.first] = i.second; | |
287 | } | |
288 | set_keys( | |
289 | ghobject_t(hobject_t(sobject_t(obj, CEPH_NOSNAP))), | |
290 | to_set); | |
291 | } | |
292 | ||
293 | void auto_set_keys(ostream &out) { | |
294 | set<string>::iterator object = rand_choose(object_name_space); | |
295 | ||
296 | map<string, string> to_set; | |
297 | unsigned amount = (rand() % 10) + 1; | |
298 | for (unsigned i = 0; i < amount; ++i) { | |
299 | set<string>::iterator key = rand_choose(key_space); | |
300 | string value = val_from_key(*object, *key); | |
301 | out << "auto_set_key " << *object << ": " << *key << " -> " | |
302 | << value << std::endl; | |
303 | to_set.insert(make_pair(*key, value)); | |
304 | } | |
305 | ||
306 | ||
307 | test_set_keys(*object, to_set); | |
308 | } | |
309 | ||
310 | void xattrs_on_object(const string &object, set<string> *out) { | |
311 | if (!xattrs.count(object)) | |
312 | return; | |
313 | const map<string, string> &xmap = xattrs.find(object)->second; | |
314 | for (map<string, string>::const_iterator i = xmap.begin(); | |
315 | i != xmap.end(); | |
316 | ++i) { | |
317 | out->insert(i->first); | |
318 | } | |
319 | } | |
320 | ||
321 | void keys_on_object(const string &object, set<string> *out) { | |
322 | if (!omap.count(object)) | |
323 | return; | |
324 | const map<string, string> &kmap = omap.find(object)->second; | |
325 | for (map<string, string>::const_iterator i = kmap.begin(); | |
326 | i != kmap.end(); | |
327 | ++i) { | |
328 | out->insert(i->first); | |
329 | } | |
330 | } | |
331 | ||
332 | void xattrs_off_object(const string &object, set<string> *out) { | |
333 | *out = key_space; | |
334 | set<string> xspace; | |
335 | xattrs_on_object(object, &xspace); | |
336 | for (set<string>::iterator i = xspace.begin(); | |
337 | i != xspace.end(); | |
338 | ++i) { | |
339 | out->erase(*i); | |
340 | } | |
341 | } | |
342 | ||
343 | void keys_off_object(const string &object, set<string> *out) { | |
344 | *out = key_space; | |
345 | set<string> kspace; | |
346 | keys_on_object(object, &kspace); | |
347 | for (set<string>::iterator i = kspace.begin(); | |
348 | i != kspace.end(); | |
349 | ++i) { | |
350 | out->erase(*i); | |
351 | } | |
352 | } | |
353 | ||
354 | int auto_check_present_xattr(ostream &out) { | |
355 | set<string>::iterator object = rand_choose(object_name_space); | |
356 | set<string> xspace; | |
357 | xattrs_on_object(*object, &xspace); | |
358 | set<string>::iterator key = rand_choose(xspace); | |
359 | if (key == xspace.end()) { | |
360 | return 1; | |
361 | } | |
362 | ||
363 | string result; | |
364 | int r = get_xattr(*object, *key, &result); | |
365 | if (!r) { | |
366 | out << "auto_check_present_key: failed to find key " | |
367 | << *key << " on object " << *object << std::endl; | |
368 | return 0; | |
369 | } | |
370 | ||
371 | if (result != xattrs[*object][*key]) { | |
372 | out << "auto_check_present_key: for key " | |
373 | << *key << " on object " << *object | |
374 | << " found value " << result << " where we should have found " | |
375 | << xattrs[*object][*key] << std::endl; | |
376 | return 0; | |
377 | } | |
378 | ||
379 | out << "auto_check_present_key: for key " | |
380 | << *key << " on object " << *object | |
381 | << " found value " << result << " where we should have found " | |
382 | << xattrs[*object][*key] << std::endl; | |
383 | return 1; | |
384 | } | |
385 | ||
386 | ||
387 | int auto_check_present_key(ostream &out) { | |
388 | set<string>::iterator object = rand_choose(object_name_space); | |
389 | set<string> kspace; | |
390 | keys_on_object(*object, &kspace); | |
391 | set<string>::iterator key = rand_choose(kspace); | |
392 | if (key == kspace.end()) { | |
393 | return 1; | |
394 | } | |
395 | ||
396 | string result; | |
397 | int r = get_key(*object, *key, &result); | |
398 | if (!r) { | |
399 | out << "auto_check_present_key: failed to find key " | |
400 | << *key << " on object " << *object << std::endl; | |
401 | return 0; | |
402 | } | |
403 | ||
404 | if (result != omap[*object][*key]) { | |
405 | out << "auto_check_present_key: for key " | |
406 | << *key << " on object " << *object | |
407 | << " found value " << result << " where we should have found " | |
408 | << omap[*object][*key] << std::endl; | |
409 | return 0; | |
410 | } | |
411 | ||
412 | out << "auto_check_present_key: for key " | |
413 | << *key << " on object " << *object | |
414 | << " found value " << result << " where we should have found " | |
415 | << omap[*object][*key] << std::endl; | |
416 | return 1; | |
417 | } | |
418 | ||
419 | int auto_check_absent_xattr(ostream &out) { | |
420 | set<string>::iterator object = rand_choose(object_name_space); | |
421 | set<string> xspace; | |
422 | xattrs_off_object(*object, &xspace); | |
423 | set<string>::iterator key = rand_choose(xspace); | |
424 | if (key == xspace.end()) { | |
425 | return 1; | |
426 | } | |
427 | ||
428 | string result; | |
429 | int r = get_xattr(*object, *key, &result); | |
430 | if (!r) { | |
431 | out << "auto_check_absent_key: did not find key " | |
432 | << *key << " on object " << *object << std::endl; | |
433 | return 1; | |
434 | } | |
435 | ||
436 | out << "auto_check_basent_key: for key " | |
437 | << *key << " on object " << *object | |
438 | << " found value " << result << " where we should have found nothing" | |
439 | << std::endl; | |
440 | return 0; | |
441 | } | |
442 | ||
443 | int auto_check_absent_key(ostream &out) { | |
444 | set<string>::iterator object = rand_choose(object_name_space); | |
445 | set<string> kspace; | |
446 | keys_off_object(*object, &kspace); | |
447 | set<string>::iterator key = rand_choose(kspace); | |
448 | if (key == kspace.end()) { | |
449 | return 1; | |
450 | } | |
451 | ||
452 | string result; | |
453 | int r = get_key(*object, *key, &result); | |
454 | if (!r) { | |
455 | out << "auto_check_absent_key: did not find key " | |
456 | << *key << " on object " << *object << std::endl; | |
457 | return 1; | |
458 | } | |
459 | ||
460 | out << "auto_check_basent_key: for key " | |
461 | << *key << " on object " << *object | |
462 | << " found value " << result << " where we should have found nothing" | |
463 | << std::endl; | |
464 | return 0; | |
465 | } | |
466 | ||
467 | void test_clone(const string &object, const string &target, ostream &out) { | |
468 | clone(object, target); | |
469 | if (!omap.count(object)) { | |
470 | out << " source missing."; | |
471 | omap.erase(target); | |
472 | } else { | |
473 | out << " source present."; | |
474 | omap[target] = omap[object]; | |
475 | } | |
476 | if (!hmap.count(object)) { | |
477 | out << " hmap source missing." << std::endl; | |
478 | hmap.erase(target); | |
479 | } else { | |
480 | out << " hmap source present." << std::endl; | |
481 | hmap[target] = hmap[object]; | |
482 | } | |
483 | if (!xattrs.count(object)) { | |
484 | out << " hmap source missing." << std::endl; | |
485 | xattrs.erase(target); | |
486 | } else { | |
487 | out << " hmap source present." << std::endl; | |
488 | xattrs[target] = xattrs[object]; | |
489 | } | |
490 | } | |
491 | ||
492 | void auto_clone_key(ostream &out) { | |
493 | set<string>::iterator object = rand_choose(object_name_space); | |
494 | set<string>::iterator target = rand_choose(object_name_space); | |
495 | while (target == object) { | |
496 | target = rand_choose(object_name_space); | |
497 | } | |
498 | out << "clone " << *object << " to " << *target; | |
499 | test_clone(*object, *target, out); | |
500 | } | |
501 | ||
502 | void test_remove_keys(const string &obj, const set<string> &to_remove) { | |
503 | for (auto &&k: to_remove) | |
504 | omap[obj].erase(k); | |
505 | remove_keys(obj, to_remove); | |
506 | } | |
507 | ||
508 | void test_remove_key(const string &obj, const string &key) { | |
509 | omap[obj].erase(key); | |
510 | remove_key(obj, key); | |
511 | } | |
512 | ||
513 | void auto_remove_keys(ostream &out) { | |
514 | set<string>::iterator object = rand_choose(object_name_space); | |
515 | set<string> kspace; | |
516 | keys_on_object(*object, &kspace); | |
517 | set<string> to_remove; | |
518 | for (unsigned i = 0; i < 3; ++i) { | |
519 | set<string>::iterator key = rand_choose(kspace); | |
520 | if (key == kspace.end()) | |
521 | continue; | |
522 | out << "removing " << *key << " from " << *object << std::endl; | |
523 | to_remove.insert(*key); | |
524 | } | |
525 | test_remove_keys(*object, to_remove); | |
526 | } | |
527 | ||
528 | void auto_remove_xattr(ostream &out) { | |
529 | set<string>::iterator object = rand_choose(object_name_space); | |
530 | set<string> kspace; | |
531 | xattrs_on_object(*object, &kspace); | |
532 | set<string>::iterator key = rand_choose(kspace); | |
533 | if (key == kspace.end()) { | |
534 | return; | |
535 | } | |
536 | out << "removing xattr " << *key << " from " << *object << std::endl; | |
537 | xattrs[*object].erase(*key); | |
538 | remove_xattr(*object, *key); | |
539 | } | |
540 | ||
541 | void auto_delete_object(ostream &out) { | |
542 | set<string>::iterator object = rand_choose(object_name_space); | |
543 | out << "auto_delete_object " << *object << std::endl; | |
544 | clear(*object); | |
545 | omap.erase(*object); | |
546 | hmap.erase(*object); | |
547 | xattrs.erase(*object); | |
548 | } | |
549 | ||
550 | void test_clear(const string &obj) { | |
551 | clear_omap(obj); | |
552 | omap.erase(obj); | |
553 | hmap.erase(obj); | |
554 | } | |
555 | ||
556 | void auto_clear_omap(ostream &out) { | |
557 | set<string>::iterator object = rand_choose(object_name_space); | |
558 | out << "auto_clear_object " << *object << std::endl; | |
559 | test_clear(*object); | |
560 | } | |
561 | ||
562 | void auto_write_header(ostream &out) { | |
563 | set<string>::iterator object = rand_choose(object_name_space); | |
564 | string header = val_from_key(*object, "HEADER"); | |
565 | out << "auto_write_header: " << *object << " -> " << header << std::endl; | |
566 | set_header(*object, header); | |
567 | hmap[*object] = header; | |
568 | } | |
569 | ||
570 | int auto_verify_header(ostream &out) { | |
571 | set<string>::iterator object = rand_choose(object_name_space); | |
572 | out << "verify_header: " << *object << " "; | |
573 | string header; | |
574 | int r = get_header(*object, &header); | |
575 | if (r < 0) { | |
576 | ceph_abort(); | |
577 | } | |
578 | if (header.size() == 0) { | |
579 | if (hmap.count(*object)) { | |
580 | out << " failed to find header " << hmap[*object] << std::endl; | |
581 | return 0; | |
582 | } else { | |
583 | out << " found no header" << std::endl; | |
584 | return 1; | |
585 | } | |
586 | } | |
587 | ||
588 | if (!hmap.count(*object)) { | |
589 | out << " found header " << header << " should have been empty" | |
590 | << std::endl; | |
591 | return 0; | |
592 | } else if (header == hmap[*object]) { | |
593 | out << " found correct header " << header << std::endl; | |
594 | return 1; | |
595 | } else { | |
596 | out << " found incorrect header " << header | |
597 | << " where we should have found " << hmap[*object] << std::endl; | |
598 | return 0; | |
599 | } | |
600 | } | |
601 | ||
602 | void verify_keys(const std::string &obj, ostream &out) { | |
603 | set<string> in_db; | |
604 | ObjectMap::ObjectMapIterator iter = db->get_iterator( | |
605 | ghobject_t(hobject_t(sobject_t(obj, CEPH_NOSNAP)))); | |
606 | for (iter->seek_to_first(); iter->valid(); iter->next()) { | |
607 | in_db.insert(iter->key()); | |
608 | } | |
609 | bool err = false; | |
610 | for (auto &&i: omap[obj]) { | |
611 | if (!in_db.count(i.first)) { | |
612 | out << __func__ << ": obj " << obj << " missing key " | |
613 | << i.first << std::endl; | |
614 | err = true; | |
615 | } else { | |
616 | in_db.erase(i.first); | |
617 | } | |
618 | } | |
619 | if (!in_db.empty()) { | |
620 | out << __func__ << ": obj " << obj << " found extra keys " | |
621 | << in_db << std::endl; | |
622 | err = true; | |
623 | } | |
624 | ASSERT_FALSE(err); | |
625 | } | |
626 | ||
627 | void auto_verify_objects(ostream &out) { | |
628 | for (auto &&i: omap) { | |
629 | verify_keys(i.first, out); | |
630 | } | |
631 | } | |
632 | }; | |
633 | ||
634 | class ObjectMapTest : public ::testing::Test { | |
635 | public: | |
636 | boost::scoped_ptr< ObjectMap > db; | |
637 | ObjectMapTester tester; | |
638 | void SetUp() override { | |
639 | char *path = getenv("OBJECT_MAP_PATH"); | |
640 | if (!path) { | |
641 | db.reset(new DBObjectMap(g_ceph_context, new KeyValueDBMemory())); | |
642 | tester.db = db.get(); | |
643 | return; | |
644 | } | |
645 | ||
646 | string strpath(path); | |
647 | ||
648 | cerr << "using path " << strpath << std::endl; | |
649 | KeyValueDB *store = KeyValueDB::create(g_ceph_context, "leveldb", strpath); | |
650 | assert(!store->create_and_open(cerr)); | |
651 | ||
652 | db.reset(new DBObjectMap(g_ceph_context, store)); | |
653 | tester.db = db.get(); | |
654 | } | |
655 | ||
656 | void TearDown() override { | |
657 | std::cerr << "Checking..." << std::endl; | |
658 | ASSERT_EQ(0, db->check(std::cerr)); | |
659 | } | |
660 | }; | |
661 | ||
662 | ||
663 | int main(int argc, char **argv) { | |
664 | vector<const char*> args; | |
665 | argv_to_vec(argc, (const char **)argv, args); | |
666 | ||
667 | auto cct = global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, | |
668 | CODE_ENVIRONMENT_UTILITY, 0); | |
669 | common_init_finish(g_ceph_context); | |
670 | ::testing::InitGoogleTest(&argc, argv); | |
671 | return RUN_ALL_TESTS(); | |
672 | } | |
673 | ||
674 | TEST_F(ObjectMapTest, CreateOneObject) { | |
675 | ghobject_t hoid(hobject_t(sobject_t("foo", CEPH_NOSNAP)), 100, shard_id_t(0)); | |
676 | map<string, bufferlist> to_set; | |
677 | string key("test"); | |
678 | string val("test_val"); | |
679 | bufferptr bp(val.c_str(), val.size()); | |
680 | bufferlist bl; | |
681 | bl.append(bp); | |
682 | to_set.insert(make_pair(key, bl)); | |
31f18b77 | 683 | ASSERT_EQ(db->set_keys(hoid, to_set), 0); |
7c673cae FG |
684 | |
685 | map<string, bufferlist> got; | |
686 | set<string> to_get; | |
687 | to_get.insert(key); | |
688 | to_get.insert("not there"); | |
689 | db->get_values(hoid, to_get, &got); | |
690 | ASSERT_EQ(got.size(), (unsigned)1); | |
691 | ASSERT_EQ(string(got[key].c_str(), got[key].length()), val); | |
692 | ||
693 | bufferlist header; | |
694 | got.clear(); | |
695 | db->get(hoid, &header, &got); | |
696 | ASSERT_EQ(got.size(), (unsigned)1); | |
697 | ASSERT_EQ(string(got[key].c_str(), got[key].length()), val); | |
698 | ASSERT_EQ(header.length(), (unsigned)0); | |
699 | ||
700 | db->rm_keys(hoid, to_get); | |
701 | got.clear(); | |
702 | db->get(hoid, &header, &got); | |
703 | ASSERT_EQ(got.size(), (unsigned)0); | |
704 | ||
705 | map<string, bufferlist> attrs; | |
706 | attrs["attr1"] = bl; | |
707 | db->set_xattrs(hoid, attrs); | |
708 | ||
709 | db->set_header(hoid, bl); | |
710 | ||
711 | db->clear_keys_header(hoid); | |
712 | set<string> attrs_got; | |
713 | db->get_all_xattrs(hoid, &attrs_got); | |
714 | ASSERT_EQ(attrs_got.size(), 1U); | |
715 | ASSERT_EQ(*(attrs_got.begin()), "attr1"); | |
716 | db->get(hoid, &header, &got); | |
717 | ASSERT_EQ(got.size(), (unsigned)0); | |
718 | ASSERT_EQ(header.length(), 0U); | |
719 | got.clear(); | |
720 | ||
721 | db->clear(hoid); | |
722 | db->get(hoid, &header, &got); | |
723 | ASSERT_EQ(got.size(), (unsigned)0); | |
724 | attrs_got.clear(); | |
725 | db->get_all_xattrs(hoid, &attrs_got); | |
726 | ASSERT_EQ(attrs_got.size(), 0U); | |
727 | } | |
728 | ||
729 | TEST_F(ObjectMapTest, CloneOneObject) { | |
730 | ghobject_t hoid(hobject_t(sobject_t("foo", CEPH_NOSNAP)), 200, shard_id_t(0)); | |
731 | ghobject_t hoid2(hobject_t(sobject_t("foo2", CEPH_NOSNAP)), 201, shard_id_t(1)); | |
732 | ||
733 | tester.set_key(hoid, "foo", "bar"); | |
734 | tester.set_key(hoid, "foo2", "bar2"); | |
735 | string result; | |
736 | int r = tester.get_key(hoid, "foo", &result); | |
737 | ASSERT_EQ(r, 1); | |
738 | ASSERT_EQ(result, "bar"); | |
739 | ||
740 | db->clone(hoid, hoid2); | |
741 | r = tester.get_key(hoid, "foo", &result); | |
742 | ASSERT_EQ(r, 1); | |
743 | ASSERT_EQ(result, "bar"); | |
744 | r = tester.get_key(hoid2, "foo", &result); | |
745 | ASSERT_EQ(r, 1); | |
746 | ASSERT_EQ(result, "bar"); | |
747 | ||
748 | tester.remove_key(hoid, "foo"); | |
749 | r = tester.get_key(hoid2, "foo", &result); | |
750 | ASSERT_EQ(r, 1); | |
751 | ASSERT_EQ(result, "bar"); | |
752 | r = tester.get_key(hoid, "foo", &result); | |
753 | ASSERT_EQ(r, 0); | |
754 | r = tester.get_key(hoid, "foo2", &result); | |
755 | ASSERT_EQ(r, 1); | |
756 | ASSERT_EQ(result, "bar2"); | |
757 | ||
758 | tester.set_key(hoid, "foo", "baz"); | |
759 | tester.remove_key(hoid, "foo"); | |
760 | r = tester.get_key(hoid, "foo", &result); | |
761 | ASSERT_EQ(r, 0); | |
762 | ||
763 | tester.set_key(hoid, "foo2", "baz"); | |
764 | tester.remove_key(hoid, "foo2"); | |
765 | r = tester.get_key(hoid, "foo2", &result); | |
766 | ASSERT_EQ(r, 0); | |
767 | ||
768 | map<string, bufferlist> got; | |
769 | bufferlist header; | |
770 | ||
771 | got.clear(); | |
772 | db->clear(hoid); | |
773 | db->get(hoid, &header, &got); | |
774 | ASSERT_EQ(got.size(), (unsigned)0); | |
775 | ||
776 | got.clear(); | |
777 | r = db->clear(hoid2); | |
778 | ASSERT_EQ(0, r); | |
779 | db->get(hoid2, &header, &got); | |
780 | ASSERT_EQ(got.size(), (unsigned)0); | |
781 | ||
782 | tester.set_key(hoid, "baz", "bar"); | |
783 | got.clear(); | |
784 | db->get(hoid, &header, &got); | |
785 | ASSERT_EQ(got.size(), (unsigned)1); | |
786 | db->clear(hoid); | |
787 | db->clear(hoid2); | |
788 | } | |
789 | ||
790 | TEST_F(ObjectMapTest, OddEvenClone) { | |
791 | ghobject_t hoid(hobject_t(sobject_t("foo", CEPH_NOSNAP))); | |
792 | ghobject_t hoid2(hobject_t(sobject_t("foo2", CEPH_NOSNAP))); | |
793 | ||
794 | for (unsigned i = 0; i < 1000; ++i) { | |
795 | tester.set_key(hoid, "foo" + num_str(i), "bar" + num_str(i)); | |
796 | } | |
797 | ||
798 | db->clone(hoid, hoid2); | |
799 | ||
800 | int r = 0; | |
801 | for (unsigned i = 0; i < 1000; ++i) { | |
802 | string result; | |
803 | r = tester.get_key(hoid, "foo" + num_str(i), &result); | |
804 | ASSERT_EQ(1, r); | |
805 | ASSERT_EQ("bar" + num_str(i), result); | |
806 | r = tester.get_key(hoid2, "foo" + num_str(i), &result); | |
807 | ASSERT_EQ(1, r); | |
808 | ASSERT_EQ("bar" + num_str(i), result); | |
809 | ||
810 | if (i % 2) { | |
811 | tester.remove_key(hoid, "foo" + num_str(i)); | |
812 | } else { | |
813 | tester.remove_key(hoid2, "foo" + num_str(i)); | |
814 | } | |
815 | } | |
816 | ||
817 | for (unsigned i = 0; i < 1000; ++i) { | |
818 | string result; | |
819 | string result2; | |
820 | r = tester.get_key(hoid, "foo" + num_str(i), &result); | |
821 | int r2 = tester.get_key(hoid2, "foo" + num_str(i), &result2); | |
822 | if (i % 2) { | |
823 | ASSERT_EQ(0, r); | |
824 | ASSERT_EQ(1, r2); | |
825 | ASSERT_EQ("bar" + num_str(i), result2); | |
826 | } else { | |
827 | ASSERT_EQ(0, r2); | |
828 | ASSERT_EQ(1, r); | |
829 | ASSERT_EQ("bar" + num_str(i), result); | |
830 | } | |
831 | } | |
832 | ||
833 | { | |
834 | ObjectMap::ObjectMapIterator iter = db->get_iterator(hoid); | |
835 | iter->seek_to_first(); | |
836 | for (unsigned i = 0; i < 1000; ++i) { | |
837 | if (!(i % 2)) { | |
838 | ASSERT_TRUE(iter->valid()); | |
839 | ASSERT_EQ("foo" + num_str(i), iter->key()); | |
840 | iter->next(); | |
841 | } | |
842 | } | |
843 | } | |
844 | ||
845 | { | |
846 | ObjectMap::ObjectMapIterator iter2 = db->get_iterator(hoid2); | |
847 | iter2->seek_to_first(); | |
848 | for (unsigned i = 0; i < 1000; ++i) { | |
849 | if (i % 2) { | |
850 | ASSERT_TRUE(iter2->valid()); | |
851 | ASSERT_EQ("foo" + num_str(i), iter2->key()); | |
852 | iter2->next(); | |
853 | } | |
854 | } | |
855 | } | |
856 | ||
857 | db->clear(hoid); | |
858 | db->clear(hoid2); | |
859 | } | |
860 | ||
861 | TEST_F(ObjectMapTest, Rename) { | |
862 | ghobject_t hoid(hobject_t(sobject_t("foo", CEPH_NOSNAP))); | |
863 | ghobject_t hoid2(hobject_t(sobject_t("foo2", CEPH_NOSNAP))); | |
864 | ||
865 | for (unsigned i = 0; i < 1000; ++i) { | |
866 | tester.set_key(hoid, "foo" + num_str(i), "bar" + num_str(i)); | |
867 | } | |
868 | ||
869 | db->rename(hoid, hoid2); | |
870 | // Verify rename where target exists | |
871 | db->clone(hoid2, hoid); | |
872 | db->rename(hoid, hoid2); | |
873 | ||
874 | int r = 0; | |
875 | for (unsigned i = 0; i < 1000; ++i) { | |
876 | string result; | |
877 | r = tester.get_key(hoid2, "foo" + num_str(i), &result); | |
878 | ASSERT_EQ(1, r); | |
879 | ASSERT_EQ("bar" + num_str(i), result); | |
880 | ||
881 | if (i % 2) { | |
882 | tester.remove_key(hoid2, "foo" + num_str(i)); | |
883 | } | |
884 | } | |
885 | ||
886 | for (unsigned i = 0; i < 1000; ++i) { | |
887 | string result; | |
888 | r = tester.get_key(hoid2, "foo" + num_str(i), &result); | |
889 | if (i % 2) { | |
890 | ASSERT_EQ(0, r); | |
891 | } else { | |
892 | ASSERT_EQ(1, r); | |
893 | ASSERT_EQ("bar" + num_str(i), result); | |
894 | } | |
895 | } | |
896 | ||
897 | { | |
898 | ObjectMap::ObjectMapIterator iter = db->get_iterator(hoid2); | |
899 | iter->seek_to_first(); | |
900 | for (unsigned i = 0; i < 1000; ++i) { | |
901 | if (!(i % 2)) { | |
902 | ASSERT_TRUE(iter->valid()); | |
903 | ASSERT_EQ("foo" + num_str(i), iter->key()); | |
904 | iter->next(); | |
905 | } | |
906 | } | |
907 | } | |
908 | ||
909 | db->clear(hoid2); | |
910 | } | |
911 | ||
912 | TEST_F(ObjectMapTest, OddEvenOldClone) { | |
913 | ghobject_t hoid(hobject_t(sobject_t("foo", CEPH_NOSNAP))); | |
914 | ghobject_t hoid2(hobject_t(sobject_t("foo2", CEPH_NOSNAP))); | |
915 | ||
916 | for (unsigned i = 0; i < 1000; ++i) { | |
917 | tester.set_key(hoid, "foo" + num_str(i), "bar" + num_str(i)); | |
918 | } | |
919 | ||
920 | db->legacy_clone(hoid, hoid2); | |
921 | ||
922 | int r = 0; | |
923 | for (unsigned i = 0; i < 1000; ++i) { | |
924 | string result; | |
925 | r = tester.get_key(hoid, "foo" + num_str(i), &result); | |
926 | ASSERT_EQ(1, r); | |
927 | ASSERT_EQ("bar" + num_str(i), result); | |
928 | r = tester.get_key(hoid2, "foo" + num_str(i), &result); | |
929 | ASSERT_EQ(1, r); | |
930 | ASSERT_EQ("bar" + num_str(i), result); | |
931 | ||
932 | if (i % 2) { | |
933 | tester.remove_key(hoid, "foo" + num_str(i)); | |
934 | } else { | |
935 | tester.remove_key(hoid2, "foo" + num_str(i)); | |
936 | } | |
937 | } | |
938 | ||
939 | for (unsigned i = 0; i < 1000; ++i) { | |
940 | string result; | |
941 | string result2; | |
942 | r = tester.get_key(hoid, "foo" + num_str(i), &result); | |
943 | int r2 = tester.get_key(hoid2, "foo" + num_str(i), &result2); | |
944 | if (i % 2) { | |
945 | ASSERT_EQ(0, r); | |
946 | ASSERT_EQ(1, r2); | |
947 | ASSERT_EQ("bar" + num_str(i), result2); | |
948 | } else { | |
949 | ASSERT_EQ(0, r2); | |
950 | ASSERT_EQ(1, r); | |
951 | ASSERT_EQ("bar" + num_str(i), result); | |
952 | } | |
953 | } | |
954 | ||
955 | { | |
956 | ObjectMap::ObjectMapIterator iter = db->get_iterator(hoid); | |
957 | iter->seek_to_first(); | |
958 | for (unsigned i = 0; i < 1000; ++i) { | |
959 | if (!(i % 2)) { | |
960 | ASSERT_TRUE(iter->valid()); | |
961 | ASSERT_EQ("foo" + num_str(i), iter->key()); | |
962 | iter->next(); | |
963 | } | |
964 | } | |
965 | } | |
966 | ||
967 | { | |
968 | ObjectMap::ObjectMapIterator iter2 = db->get_iterator(hoid2); | |
969 | iter2->seek_to_first(); | |
970 | for (unsigned i = 0; i < 1000; ++i) { | |
971 | if (i % 2) { | |
972 | ASSERT_TRUE(iter2->valid()); | |
973 | ASSERT_EQ("foo" + num_str(i), iter2->key()); | |
974 | iter2->next(); | |
975 | } | |
976 | } | |
977 | } | |
978 | ||
979 | db->clear(hoid); | |
980 | db->clear(hoid2); | |
981 | } | |
982 | ||
983 | TEST_F(ObjectMapTest, RandomTest) { | |
984 | tester.def_init(); | |
985 | for (unsigned i = 0; i < 5000; ++i) { | |
986 | unsigned val = rand(); | |
987 | val <<= 8; | |
988 | val %= 100; | |
989 | if (!(i%100)) | |
990 | std::cout << "on op " << i | |
991 | << " val is " << val << std::endl; | |
992 | ||
993 | if (val < 7) { | |
994 | tester.auto_write_header(std::cerr); | |
995 | } else if (val < 14) { | |
996 | ASSERT_TRUE(tester.auto_verify_header(std::cerr)); | |
997 | } else if (val < 30) { | |
998 | tester.auto_set_keys(std::cerr); | |
999 | } else if (val < 42) { | |
1000 | tester.auto_set_xattr(std::cerr); | |
1001 | } else if (val < 55) { | |
1002 | ASSERT_TRUE(tester.auto_check_present_key(std::cerr)); | |
1003 | } else if (val < 62) { | |
1004 | ASSERT_TRUE(tester.auto_check_present_xattr(std::cerr)); | |
1005 | } else if (val < 70) { | |
1006 | ASSERT_TRUE(tester.auto_check_absent_key(std::cerr)); | |
1007 | } else if (val < 72) { | |
1008 | ASSERT_TRUE(tester.auto_check_absent_xattr(std::cerr)); | |
1009 | } else if (val < 73) { | |
1010 | tester.auto_clear_omap(std::cerr); | |
1011 | } else if (val < 76) { | |
1012 | tester.auto_delete_object(std::cerr); | |
1013 | } else if (val < 85) { | |
1014 | tester.auto_clone_key(std::cerr); | |
1015 | } else if (val < 92) { | |
1016 | tester.auto_remove_xattr(std::cerr); | |
1017 | } else { | |
1018 | tester.auto_remove_keys(std::cerr); | |
1019 | } | |
1020 | ||
1021 | if (i % 500) { | |
1022 | tester.auto_verify_objects(std::cerr); | |
1023 | } | |
1024 | } | |
1025 | } | |
1026 | ||
1027 | TEST_F(ObjectMapTest, RandomTestNoDeletesXattrs) { | |
1028 | tester.def_init(); | |
1029 | for (unsigned i = 0; i < 5000; ++i) { | |
1030 | unsigned val = rand(); | |
1031 | val <<= 8; | |
1032 | val %= 100; | |
1033 | if (!(i%100)) | |
1034 | std::cout << "on op " << i | |
1035 | << " val is " << val << std::endl; | |
1036 | ||
1037 | if (val < 45) { | |
1038 | tester.auto_set_keys(std::cerr); | |
1039 | } else if (val < 90) { | |
1040 | tester.auto_remove_keys(std::cerr); | |
1041 | } else { | |
1042 | tester.auto_clone_key(std::cerr); | |
1043 | } | |
1044 | ||
1045 | if (i % 500) { | |
1046 | tester.auto_verify_objects(std::cerr); | |
1047 | } | |
1048 | } | |
1049 | } | |
1050 | ||
1051 | string num_to_key(unsigned i) { | |
1052 | char buf[100]; | |
1053 | int ret = snprintf(buf, sizeof(buf), "%010u", i); | |
1054 | assert(ret > 0); | |
1055 | return string(buf, ret); | |
1056 | } | |
1057 | ||
1058 | TEST_F(ObjectMapTest, TestMergeNewCompleteContainBug) { | |
1059 | /* This test exploits a bug in kraken and earlier where merge_new_complete | |
1060 | * could miss complete entries fully contained by a new entry. To get this | |
1061 | * to actually result in an incorrect return value, you need to remove at | |
1062 | * least two values, one before a complete region, and one which occurs in | |
1063 | * the parent after the complete region (but within 20 not yet completed | |
1064 | * parent points of the first value). | |
1065 | */ | |
1066 | for (unsigned i = 10; i < 160; i+=2) { | |
1067 | tester.test_set_key("foo", num_to_key(i), "asdf"); | |
1068 | } | |
1069 | tester.test_clone("foo", "foo2", std::cout); | |
1070 | tester.test_clear("foo"); | |
1071 | ||
1072 | tester.test_set_key("foo2", num_to_key(15), "asdf"); | |
1073 | tester.test_set_key("foo2", num_to_key(13), "asdf"); | |
1074 | tester.test_set_key("foo2", num_to_key(57), "asdf"); | |
1075 | ||
1076 | tester.test_remove_key("foo2", num_to_key(15)); | |
1077 | ||
1078 | set<string> to_remove; | |
1079 | to_remove.insert(num_to_key(13)); | |
1080 | to_remove.insert(num_to_key(58)); | |
1081 | to_remove.insert(num_to_key(60)); | |
1082 | to_remove.insert(num_to_key(62)); | |
1083 | tester.test_remove_keys("foo2", to_remove); | |
1084 | ||
1085 | tester.verify_keys("foo2", std::cout); | |
1086 | ASSERT_EQ(tester.get_key("foo2", num_to_key(10), nullptr), 1); | |
1087 | ASSERT_EQ(tester.get_key("foo2", num_to_key(1), nullptr), 0); | |
1088 | ASSERT_EQ(tester.get_key("foo2", num_to_key(56), nullptr), 1); | |
1089 | // this one triggers the bug | |
1090 | ASSERT_EQ(tester.get_key("foo2", num_to_key(58), nullptr), 0); | |
1091 | } | |
1092 | ||
1093 | TEST_F(ObjectMapTest, TestIterateBug18533) { | |
1094 | /* This test starts with the one immediately above to create a pair of | |
1095 | * complete regions where one contains the other. Then, it deletes the | |
1096 | * key at the start of the contained region. The logic in next_parent() | |
1097 | * skips ahead to the end of the contained region, and we start copying | |
1098 | * values down again from the parent into the child -- including some | |
1099 | * that had actually been deleted. I think this works for any removal | |
1100 | * within the outer complete region after the start of the contained | |
1101 | * region. | |
1102 | */ | |
1103 | for (unsigned i = 10; i < 160; i+=2) { | |
1104 | tester.test_set_key("foo", num_to_key(i), "asdf"); | |
1105 | } | |
1106 | tester.test_clone("foo", "foo2", std::cout); | |
1107 | tester.test_clear("foo"); | |
1108 | ||
1109 | tester.test_set_key("foo2", num_to_key(15), "asdf"); | |
1110 | tester.test_set_key("foo2", num_to_key(13), "asdf"); | |
1111 | tester.test_set_key("foo2", num_to_key(57), "asdf"); | |
1112 | tester.test_set_key("foo2", num_to_key(91), "asdf"); | |
1113 | ||
1114 | tester.test_remove_key("foo2", num_to_key(15)); | |
1115 | ||
1116 | set<string> to_remove; | |
1117 | to_remove.insert(num_to_key(13)); | |
1118 | to_remove.insert(num_to_key(58)); | |
1119 | to_remove.insert(num_to_key(60)); | |
1120 | to_remove.insert(num_to_key(62)); | |
1121 | to_remove.insert(num_to_key(82)); | |
1122 | to_remove.insert(num_to_key(84)); | |
1123 | tester.test_remove_keys("foo2", to_remove); | |
1124 | ||
1125 | //tester.test_remove_key("foo2", num_to_key(15)); also does the trick | |
1126 | tester.test_remove_key("foo2", num_to_key(80)); | |
1127 | ||
1128 | // the iterator in verify_keys will return an extra value | |
1129 | tester.verify_keys("foo2", std::cout); | |
1130 | } | |
1131 |