]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- |
2 | // vim: ts=8 sw=2 smarttab | |
3 | /* | |
4 | * Ceph - scalable distributed file system | |
5 | * | |
6 | * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net> | |
7 | * | |
8 | * This is free software; you can redistribute it and/or | |
9 | * modify it under the terms of the GNU Lesser General Public | |
10 | * License version 2.1, as published by the Free Software | |
11 | * Foundation. See file COPYING. | |
12 | * | |
13 | */ | |
14 | ||
15 | #include <stdio.h> | |
16 | #include <string.h> | |
17 | #include <iostream> | |
18 | #include <time.h> | |
19 | #include <sys/mount.h> | |
20 | #include "kv/KeyValueDB.h" | |
f67539c2 | 21 | #include "kv/RocksDBStore.h" |
7c673cae FG |
22 | #include "include/Context.h" |
23 | #include "common/ceph_argparse.h" | |
24 | #include "global/global_init.h" | |
7c673cae FG |
25 | #include "common/Cond.h" |
26 | #include "common/errno.h" | |
27 | #include "include/stringify.h" | |
28 | #include <gtest/gtest.h> | |
29 | ||
7c673cae FG |
30 | class KVTest : public ::testing::TestWithParam<const char*> { |
31 | public: | |
32 | boost::scoped_ptr<KeyValueDB> db; | |
33 | ||
34 | KVTest() : db(0) {} | |
35 | ||
11fdf7f2 TL |
36 | string _bl_to_str(bufferlist val) { |
37 | string str(val.c_str(), val.length()); | |
38 | return str; | |
39 | } | |
40 | ||
7c673cae FG |
41 | void rm_r(string path) { |
42 | string cmd = string("rm -r ") + path; | |
43 | cout << "==> " << cmd << std::endl; | |
44 | int r = ::system(cmd.c_str()); | |
45 | if (r) { | |
46 | cerr << "failed with exit code " << r | |
47 | << ", continuing anyway" << std::endl; | |
48 | } | |
49 | } | |
50 | ||
51 | void init() { | |
52 | cout << "Creating " << string(GetParam()) << "\n"; | |
53 | db.reset(KeyValueDB::create(g_ceph_context, string(GetParam()), | |
54 | "kv_test_temp_dir")); | |
55 | } | |
56 | void fini() { | |
57 | db.reset(NULL); | |
58 | } | |
59 | ||
60 | void SetUp() override { | |
61 | int r = ::mkdir("kv_test_temp_dir", 0777); | |
62 | if (r < 0 && errno != EEXIST) { | |
63 | r = -errno; | |
64 | cerr << __func__ << ": unable to create kv_test_temp_dir: " | |
65 | << cpp_strerror(r) << std::endl; | |
66 | return; | |
67 | } | |
68 | init(); | |
69 | } | |
70 | void TearDown() override { | |
71 | fini(); | |
72 | rm_r("kv_test_temp_dir"); | |
73 | } | |
74 | }; | |
75 | ||
76 | TEST_P(KVTest, OpenClose) { | |
77 | ASSERT_EQ(0, db->create_and_open(cout)); | |
f67539c2 TL |
78 | db->close(); |
79 | db->open(cout); | |
7c673cae FG |
80 | fini(); |
81 | } | |
82 | ||
83 | TEST_P(KVTest, OpenCloseReopenClose) { | |
84 | ASSERT_EQ(0, db->create_and_open(cout)); | |
85 | fini(); | |
86 | init(); | |
87 | ASSERT_EQ(0, db->open(cout)); | |
88 | fini(); | |
89 | } | |
90 | ||
91 | /* | |
92 | * Basic write and read test case in same database session. | |
93 | */ | |
94 | TEST_P(KVTest, OpenWriteRead) { | |
95 | ASSERT_EQ(0, db->create_and_open(cout)); | |
96 | { | |
97 | KeyValueDB::Transaction t = db->get_transaction(); | |
98 | bufferlist value; | |
99 | value.append("value"); | |
100 | t->set("prefix", "key", value); | |
101 | value.clear(); | |
102 | value.append("value2"); | |
103 | t->set("prefix", "key2", value); | |
104 | value.clear(); | |
105 | value.append("value3"); | |
106 | t->set("prefix", "key3", value); | |
107 | db->submit_transaction_sync(t); | |
108 | ||
109 | bufferlist v1, v2; | |
110 | ASSERT_EQ(0, db->get("prefix", "key", &v1)); | |
111 | ASSERT_EQ(v1.length(), 5u); | |
112 | (v1.c_str())[v1.length()] = 0x0; | |
113 | ASSERT_EQ(std::string(v1.c_str()), std::string("value")); | |
114 | ASSERT_EQ(0, db->get("prefix", "key2", &v2)); | |
115 | ASSERT_EQ(v2.length(), 6u); | |
116 | (v2.c_str())[v2.length()] = 0x0; | |
117 | ASSERT_EQ(std::string(v2.c_str()), std::string("value2")); | |
118 | } | |
119 | fini(); | |
120 | } | |
121 | ||
122 | TEST_P(KVTest, PutReopen) { | |
123 | ASSERT_EQ(0, db->create_and_open(cout)); | |
124 | { | |
125 | KeyValueDB::Transaction t = db->get_transaction(); | |
126 | bufferlist value; | |
127 | value.append("value"); | |
128 | t->set("prefix", "key", value); | |
129 | t->set("prefix", "key2", value); | |
130 | t->set("prefix", "key3", value); | |
131 | db->submit_transaction_sync(t); | |
132 | } | |
133 | fini(); | |
134 | ||
135 | init(); | |
136 | ASSERT_EQ(0, db->open(cout)); | |
137 | { | |
138 | bufferlist v1, v2; | |
139 | ASSERT_EQ(0, db->get("prefix", "key", &v1)); | |
140 | ASSERT_EQ(v1.length(), 5u); | |
141 | ASSERT_EQ(0, db->get("prefix", "key2", &v2)); | |
142 | ASSERT_EQ(v2.length(), 5u); | |
143 | } | |
144 | { | |
145 | KeyValueDB::Transaction t = db->get_transaction(); | |
146 | t->rmkey("prefix", "key"); | |
147 | t->rmkey("prefix", "key3"); | |
148 | db->submit_transaction_sync(t); | |
149 | } | |
150 | fini(); | |
151 | ||
152 | init(); | |
153 | ASSERT_EQ(0, db->open(cout)); | |
154 | { | |
155 | bufferlist v1, v2, v3; | |
156 | ASSERT_EQ(-ENOENT, db->get("prefix", "key", &v1)); | |
157 | ASSERT_EQ(0, db->get("prefix", "key2", &v2)); | |
158 | ASSERT_EQ(v2.length(), 5u); | |
159 | ASSERT_EQ(-ENOENT, db->get("prefix", "key3", &v3)); | |
160 | } | |
161 | fini(); | |
162 | } | |
163 | ||
164 | TEST_P(KVTest, BenchCommit) { | |
165 | int n = 1024; | |
166 | ASSERT_EQ(0, db->create_and_open(cout)); | |
167 | utime_t start = ceph_clock_now(); | |
168 | { | |
169 | cout << "priming" << std::endl; | |
170 | // prime | |
171 | bufferlist big; | |
172 | bufferptr bp(1048576); | |
173 | bp.zero(); | |
174 | big.append(bp); | |
175 | for (int i=0; i<30; ++i) { | |
176 | KeyValueDB::Transaction t = db->get_transaction(); | |
177 | t->set("prefix", "big" + stringify(i), big); | |
178 | db->submit_transaction_sync(t); | |
179 | } | |
180 | } | |
181 | cout << "now doing small writes" << std::endl; | |
182 | bufferlist data; | |
183 | bufferptr bp(1024); | |
184 | bp.zero(); | |
185 | data.append(bp); | |
186 | for (int i=0; i<n; ++i) { | |
187 | KeyValueDB::Transaction t = db->get_transaction(); | |
188 | t->set("prefix", "key" + stringify(i), data); | |
189 | db->submit_transaction_sync(t); | |
190 | } | |
191 | utime_t end = ceph_clock_now(); | |
192 | utime_t dur = end - start; | |
193 | cout << n << " commits in " << dur << ", avg latency " << (dur / (double)n) | |
194 | << std::endl; | |
195 | fini(); | |
196 | } | |
197 | ||
198 | struct AppendMOP : public KeyValueDB::MergeOperator { | |
199 | void merge_nonexistent( | |
200 | const char *rdata, size_t rlen, std::string *new_value) override { | |
201 | *new_value = "?" + std::string(rdata, rlen); | |
202 | } | |
203 | void merge( | |
204 | const char *ldata, size_t llen, | |
205 | const char *rdata, size_t rlen, | |
206 | std::string *new_value) override { | |
7c673cae FG |
207 | *new_value = std::string(ldata, llen) + std::string(rdata, rlen); |
208 | } | |
209 | // We use each operator name and each prefix to construct the | |
210 | // overall RocksDB operator name for consistency check at open time. | |
91327a77 | 211 | const char *name() const override { |
7c673cae FG |
212 | return "Append"; |
213 | } | |
214 | }; | |
215 | ||
216 | string tostr(bufferlist& b) { | |
217 | return string(b.c_str(),b.length()); | |
218 | } | |
219 | ||
220 | TEST_P(KVTest, Merge) { | |
221 | shared_ptr<KeyValueDB::MergeOperator> p(new AppendMOP); | |
222 | int r = db->set_merge_operator("A",p); | |
223 | if (r < 0) | |
224 | return; // No merge operators for this database type | |
225 | ASSERT_EQ(0, db->create_and_open(cout)); | |
226 | { | |
227 | KeyValueDB::Transaction t = db->get_transaction(); | |
228 | bufferlist v1, v2, v3; | |
229 | v1.append(string("1")); | |
230 | v2.append(string("2")); | |
231 | v3.append(string("3")); | |
232 | t->set("P", "K1", v1); | |
233 | t->set("A", "A1", v2); | |
234 | t->rmkey("A", "A2"); | |
235 | t->merge("A", "A2", v3); | |
236 | db->submit_transaction_sync(t); | |
237 | } | |
238 | { | |
239 | bufferlist v1, v2, v3; | |
240 | ASSERT_EQ(0, db->get("P", "K1", &v1)); | |
241 | ASSERT_EQ(tostr(v1), "1"); | |
242 | ASSERT_EQ(0, db->get("A", "A1", &v2)); | |
243 | ASSERT_EQ(tostr(v2), "2"); | |
244 | ASSERT_EQ(0, db->get("A", "A2", &v3)); | |
245 | ASSERT_EQ(tostr(v3), "?3"); | |
246 | } | |
247 | { | |
248 | KeyValueDB::Transaction t = db->get_transaction(); | |
249 | bufferlist v1; | |
250 | v1.append(string("1")); | |
251 | t->merge("A", "A2", v1); | |
252 | db->submit_transaction_sync(t); | |
253 | } | |
254 | { | |
255 | bufferlist v; | |
256 | ASSERT_EQ(0, db->get("A", "A2", &v)); | |
257 | ASSERT_EQ(tostr(v), "?31"); | |
258 | } | |
259 | fini(); | |
260 | } | |
261 | ||
262 | TEST_P(KVTest, RMRange) { | |
263 | ASSERT_EQ(0, db->create_and_open(cout)); | |
264 | bufferlist value; | |
265 | value.append("value"); | |
266 | { | |
267 | KeyValueDB::Transaction t = db->get_transaction(); | |
268 | t->set("prefix", "key1", value); | |
269 | t->set("prefix", "key2", value); | |
270 | t->set("prefix", "key3", value); | |
271 | t->set("prefix", "key4", value); | |
272 | t->set("prefix", "key45", value); | |
273 | t->set("prefix", "key5", value); | |
274 | t->set("prefix", "key6", value); | |
275 | db->submit_transaction_sync(t); | |
276 | } | |
277 | ||
278 | { | |
279 | KeyValueDB::Transaction t = db->get_transaction(); | |
280 | t->set("prefix", "key7", value); | |
281 | t->set("prefix", "key8", value); | |
282 | t->rm_range_keys("prefix", "key2", "key7"); | |
283 | db->submit_transaction_sync(t); | |
284 | bufferlist v1, v2; | |
285 | ASSERT_EQ(0, db->get("prefix", "key1", &v1)); | |
286 | v1.clear(); | |
287 | ASSERT_EQ(-ENOENT, db->get("prefix", "key45", &v1)); | |
288 | ASSERT_EQ(0, db->get("prefix", "key8", &v1)); | |
289 | v1.clear(); | |
290 | ASSERT_EQ(-ENOENT, db->get("prefix", "key2", &v1)); | |
291 | ASSERT_EQ(0, db->get("prefix", "key7", &v2)); | |
292 | } | |
293 | ||
294 | { | |
295 | KeyValueDB::Transaction t = db->get_transaction(); | |
296 | t->rm_range_keys("prefix", "key", "key"); | |
297 | db->submit_transaction_sync(t); | |
298 | bufferlist v1, v2; | |
299 | ASSERT_EQ(0, db->get("prefix", "key1", &v1)); | |
300 | ASSERT_EQ(0, db->get("prefix", "key8", &v2)); | |
301 | } | |
302 | ||
303 | { | |
304 | KeyValueDB::Transaction t = db->get_transaction(); | |
305 | t->rm_range_keys("prefix", "key-", "key~"); | |
306 | db->submit_transaction_sync(t); | |
307 | bufferlist v1, v2; | |
308 | ASSERT_EQ(-ENOENT, db->get("prefix", "key1", &v1)); | |
309 | ASSERT_EQ(-ENOENT, db->get("prefix", "key8", &v2)); | |
310 | } | |
311 | ||
312 | fini(); | |
313 | } | |
314 | ||
f67539c2 TL |
315 | TEST_P(KVTest, ShardingRMRange) { |
316 | if(string(GetParam()) != "rocksdb") | |
317 | return; | |
318 | std::string cfs("O(7)="); | |
319 | ASSERT_EQ(0, db->create_and_open(cout, cfs)); | |
320 | { | |
321 | KeyValueDB::Transaction t = db->get_transaction(); | |
322 | for (size_t i = 0; i < 1000; i++) { | |
323 | bufferlist value; | |
324 | char* a; | |
325 | ASSERT_EQ(asprintf(&a, "key%3.3ld", i), 6); | |
326 | value.append(a); | |
327 | t->set("O", a, value); | |
328 | free(a); | |
329 | } | |
330 | db->submit_transaction_sync(t); | |
331 | } | |
332 | ||
333 | { | |
334 | KeyValueDB::Transaction t = db->get_transaction(); | |
335 | t->rm_range_keys("O", "key277", "key467"); | |
336 | db->submit_transaction_sync(t); | |
337 | } | |
338 | ||
339 | for (size_t i = 0; i < 1000; i++) { | |
340 | char* key; | |
341 | ASSERT_EQ(asprintf(&key, "key%3.3ld", i), 6); | |
342 | bufferlist value; | |
343 | int r = db->get("O", key, &value); | |
344 | ASSERT_EQ(r, (i >= 277 && i < 467 ? -ENOENT : 0)); | |
345 | free(key); | |
346 | } | |
347 | ||
348 | fini(); | |
349 | } | |
350 | ||
351 | ||
11fdf7f2 TL |
352 | TEST_P(KVTest, RocksDBColumnFamilyTest) { |
353 | if(string(GetParam()) != "rocksdb") | |
354 | return; | |
355 | ||
f67539c2 | 356 | std::string cfs("cf1 cf2"); |
11fdf7f2 TL |
357 | ASSERT_EQ(0, db->init(g_conf()->bluestore_rocksdb_options)); |
358 | cout << "creating two column families and opening them" << std::endl; | |
359 | ASSERT_EQ(0, db->create_and_open(cout, cfs)); | |
360 | { | |
361 | KeyValueDB::Transaction t = db->get_transaction(); | |
362 | bufferlist value; | |
363 | value.append("value"); | |
364 | cout << "write a transaction includes three keys in different CFs" << std::endl; | |
365 | t->set("prefix", "key", value); | |
366 | t->set("cf1", "key", value); | |
367 | t->set("cf2", "key2", value); | |
368 | ASSERT_EQ(0, db->submit_transaction_sync(t)); | |
369 | } | |
370 | fini(); | |
371 | ||
372 | init(); | |
373 | ASSERT_EQ(0, db->open(cout, cfs)); | |
374 | { | |
375 | bufferlist v1, v2, v3; | |
376 | cout << "reopen db and read those keys" << std::endl; | |
377 | ASSERT_EQ(0, db->get("prefix", "key", &v1)); | |
378 | ASSERT_EQ(0, _bl_to_str(v1) != "value"); | |
379 | ASSERT_EQ(0, db->get("cf1", "key", &v2)); | |
380 | ASSERT_EQ(0, _bl_to_str(v2) != "value"); | |
381 | ASSERT_EQ(0, db->get("cf2", "key2", &v3)); | |
382 | ASSERT_EQ(0, _bl_to_str(v2) != "value"); | |
383 | } | |
384 | { | |
385 | cout << "delete two keys in CFs" << std::endl; | |
386 | KeyValueDB::Transaction t = db->get_transaction(); | |
387 | t->rmkey("prefix", "key"); | |
388 | t->rmkey("cf2", "key2"); | |
389 | ASSERT_EQ(0, db->submit_transaction_sync(t)); | |
390 | } | |
391 | fini(); | |
392 | ||
393 | init(); | |
394 | ASSERT_EQ(0, db->open(cout, cfs)); | |
395 | { | |
396 | cout << "reopen db and read keys again." << std::endl; | |
397 | bufferlist v1, v2, v3; | |
398 | ASSERT_EQ(-ENOENT, db->get("prefix", "key", &v1)); | |
399 | ASSERT_EQ(0, db->get("cf1", "key", &v2)); | |
400 | ASSERT_EQ(0, _bl_to_str(v2) != "value"); | |
401 | ASSERT_EQ(-ENOENT, db->get("cf2", "key2", &v3)); | |
402 | } | |
403 | fini(); | |
404 | } | |
405 | ||
406 | TEST_P(KVTest, RocksDBIteratorTest) { | |
407 | if(string(GetParam()) != "rocksdb") | |
408 | return; | |
409 | ||
f67539c2 | 410 | std::string cfs("cf1"); |
11fdf7f2 TL |
411 | ASSERT_EQ(0, db->init(g_conf()->bluestore_rocksdb_options)); |
412 | cout << "creating one column family and opening it" << std::endl; | |
413 | ASSERT_EQ(0, db->create_and_open(cout, cfs)); | |
414 | { | |
415 | KeyValueDB::Transaction t = db->get_transaction(); | |
416 | bufferlist bl1; | |
417 | bl1.append("hello"); | |
418 | bufferlist bl2; | |
419 | bl2.append("world"); | |
420 | cout << "write some kv pairs into default and new CFs" << std::endl; | |
421 | t->set("prefix", "key1", bl1); | |
422 | t->set("prefix", "key2", bl2); | |
423 | t->set("cf1", "key1", bl1); | |
424 | t->set("cf1", "key2", bl2); | |
425 | ASSERT_EQ(0, db->submit_transaction_sync(t)); | |
426 | } | |
427 | { | |
428 | cout << "iterating the default CF" << std::endl; | |
429 | KeyValueDB::Iterator iter = db->get_iterator("prefix"); | |
430 | iter->seek_to_first(); | |
431 | ASSERT_EQ(1, iter->valid()); | |
432 | ASSERT_EQ("key1", iter->key()); | |
433 | ASSERT_EQ("hello", _bl_to_str(iter->value())); | |
434 | ASSERT_EQ(0, iter->next()); | |
435 | ASSERT_EQ(1, iter->valid()); | |
436 | ASSERT_EQ("key2", iter->key()); | |
437 | ASSERT_EQ("world", _bl_to_str(iter->value())); | |
438 | } | |
439 | { | |
440 | cout << "iterating the new CF" << std::endl; | |
441 | KeyValueDB::Iterator iter = db->get_iterator("cf1"); | |
442 | iter->seek_to_first(); | |
443 | ASSERT_EQ(1, iter->valid()); | |
444 | ASSERT_EQ("key1", iter->key()); | |
445 | ASSERT_EQ("hello", _bl_to_str(iter->value())); | |
446 | ASSERT_EQ(0, iter->next()); | |
447 | ASSERT_EQ(1, iter->valid()); | |
448 | ASSERT_EQ("key2", iter->key()); | |
449 | ASSERT_EQ("world", _bl_to_str(iter->value())); | |
450 | } | |
451 | fini(); | |
452 | } | |
453 | ||
f67539c2 TL |
454 | TEST_P(KVTest, RocksDBShardingIteratorTest) { |
455 | if(string(GetParam()) != "rocksdb") | |
456 | return; | |
457 | ||
458 | std::string cfs("A(6)"); | |
459 | ASSERT_EQ(0, db->init(g_conf()->bluestore_rocksdb_options)); | |
460 | cout << "creating one column family and opening it" << std::endl; | |
461 | ASSERT_EQ(0, db->create_and_open(cout, cfs)); | |
462 | { | |
463 | KeyValueDB::Transaction t = db->get_transaction(); | |
464 | for (int v = 100; v <= 999; v++) { | |
465 | std::string str = to_string(v); | |
466 | bufferlist val; | |
467 | val.append(str); | |
468 | t->set("A", str, val); | |
469 | } | |
470 | ASSERT_EQ(0, db->submit_transaction_sync(t)); | |
471 | } | |
472 | { | |
473 | KeyValueDB::Iterator it = db->get_iterator("A"); | |
474 | int pos = 0; | |
475 | ASSERT_EQ(it->lower_bound(to_string(pos)), 0); | |
476 | for (pos = 100; pos <= 999; pos++) { | |
477 | ASSERT_EQ(it->valid(), true); | |
478 | ASSERT_EQ(it->key(), to_string(pos)); | |
479 | ASSERT_EQ(it->value().to_str(), to_string(pos)); | |
480 | it->next(); | |
481 | } | |
482 | ASSERT_EQ(it->valid(), false); | |
483 | pos = 999; | |
484 | ASSERT_EQ(it->lower_bound(to_string(pos)), 0); | |
485 | for (pos = 999; pos >= 100; pos--) { | |
486 | ASSERT_EQ(it->valid(), true); | |
487 | ASSERT_EQ(it->key(), to_string(pos)); | |
488 | ASSERT_EQ(it->value().to_str(), to_string(pos)); | |
489 | it->prev(); | |
490 | } | |
491 | ASSERT_EQ(it->valid(), false); | |
492 | } | |
493 | fini(); | |
494 | } | |
495 | ||
11fdf7f2 TL |
496 | TEST_P(KVTest, RocksDBCFMerge) { |
497 | if(string(GetParam()) != "rocksdb") | |
498 | return; | |
499 | ||
500 | shared_ptr<KeyValueDB::MergeOperator> p(new AppendMOP); | |
501 | int r = db->set_merge_operator("cf1",p); | |
502 | if (r < 0) | |
503 | return; // No merge operators for this database type | |
f67539c2 | 504 | std::string cfs("cf1"); |
11fdf7f2 TL |
505 | ASSERT_EQ(0, db->init(g_conf()->bluestore_rocksdb_options)); |
506 | cout << "creating one column family and opening it" << std::endl; | |
507 | ASSERT_EQ(0, db->create_and_open(cout, cfs)); | |
508 | ||
509 | { | |
510 | KeyValueDB::Transaction t = db->get_transaction(); | |
511 | bufferlist v1, v2, v3; | |
512 | v1.append(string("1")); | |
513 | v2.append(string("2")); | |
514 | v3.append(string("3")); | |
515 | t->set("P", "K1", v1); | |
516 | t->set("cf1", "A1", v2); | |
517 | t->rmkey("cf1", "A2"); | |
518 | t->merge("cf1", "A2", v3); | |
519 | db->submit_transaction_sync(t); | |
520 | } | |
521 | { | |
522 | bufferlist v1, v2, v3; | |
523 | ASSERT_EQ(0, db->get("P", "K1", &v1)); | |
524 | ASSERT_EQ(tostr(v1), "1"); | |
525 | ASSERT_EQ(0, db->get("cf1", "A1", &v2)); | |
526 | ASSERT_EQ(tostr(v2), "2"); | |
527 | ASSERT_EQ(0, db->get("cf1", "A2", &v3)); | |
528 | ASSERT_EQ(tostr(v3), "?3"); | |
529 | } | |
530 | { | |
531 | KeyValueDB::Transaction t = db->get_transaction(); | |
532 | bufferlist v1; | |
533 | v1.append(string("1")); | |
534 | t->merge("cf1", "A2", v1); | |
535 | db->submit_transaction_sync(t); | |
536 | } | |
537 | { | |
538 | bufferlist v; | |
539 | ASSERT_EQ(0, db->get("cf1", "A2", &v)); | |
540 | ASSERT_EQ(tostr(v), "?31"); | |
541 | } | |
542 | fini(); | |
543 | } | |
7c673cae | 544 | |
9f95a23c TL |
545 | TEST_P(KVTest, RocksDB_estimate_size) { |
546 | if(string(GetParam()) != "rocksdb") | |
547 | GTEST_SKIP(); | |
548 | ||
f67539c2 | 549 | std::string cfs("cf1"); |
9f95a23c TL |
550 | ASSERT_EQ(0, db->init(g_conf()->bluestore_rocksdb_options)); |
551 | cout << "creating one column family and opening it" << std::endl; | |
552 | ASSERT_EQ(0, db->create_and_open(cout)); | |
553 | ||
554 | for(int test = 0; test < 20; test++) | |
555 | { | |
556 | KeyValueDB::Transaction t = db->get_transaction(); | |
557 | bufferlist v1; | |
558 | v1.append(string(1000, '1')); | |
559 | for (int i = 0; i < 100; i++) | |
560 | t->set("A", to_string(rand()%100000), v1); | |
561 | db->submit_transaction_sync(t); | |
562 | db->compact(); | |
563 | ||
564 | int64_t size_a = db->estimate_prefix_size("A",""); | |
565 | ASSERT_GT(size_a, (test + 1) * 1000 * 100 * 0.5); | |
566 | ASSERT_LT(size_a, (test + 1) * 1000 * 100 * 1.5); | |
567 | int64_t size_a1 = db->estimate_prefix_size("A","1"); | |
568 | ASSERT_GT(size_a1, (test + 1) * 1000 * 100 * 0.1 * 0.5); | |
569 | ASSERT_LT(size_a1, (test + 1) * 1000 * 100 * 0.1 * 1.5); | |
570 | int64_t size_b = db->estimate_prefix_size("B",""); | |
571 | ASSERT_EQ(size_b, 0); | |
572 | } | |
573 | ||
574 | fini(); | |
575 | } | |
576 | ||
577 | TEST_P(KVTest, RocksDB_estimate_size_column_family) { | |
578 | if(string(GetParam()) != "rocksdb") | |
579 | GTEST_SKIP(); | |
580 | ||
f67539c2 | 581 | std::string cfs("cf1"); |
9f95a23c TL |
582 | ASSERT_EQ(0, db->init(g_conf()->bluestore_rocksdb_options)); |
583 | cout << "creating one column family and opening it" << std::endl; | |
584 | ASSERT_EQ(0, db->create_and_open(cout, cfs)); | |
585 | ||
586 | for(int test = 0; test < 20; test++) | |
587 | { | |
588 | KeyValueDB::Transaction t = db->get_transaction(); | |
589 | bufferlist v1; | |
590 | v1.append(string(1000, '1')); | |
591 | for (int i = 0; i < 100; i++) | |
592 | t->set("cf1", to_string(rand()%100000), v1); | |
593 | db->submit_transaction_sync(t); | |
594 | db->compact(); | |
7c673cae | 595 | |
9f95a23c TL |
596 | int64_t size_a = db->estimate_prefix_size("cf1",""); |
597 | ASSERT_GT(size_a, (test + 1) * 1000 * 100 * 0.5); | |
598 | ASSERT_LT(size_a, (test + 1) * 1000 * 100 * 1.5); | |
599 | int64_t size_a1 = db->estimate_prefix_size("cf1","1"); | |
600 | ASSERT_GT(size_a1, (test + 1) * 1000 * 100 * 0.1 * 0.5); | |
601 | ASSERT_LT(size_a1, (test + 1) * 1000 * 100 * 0.1 * 1.5); | |
602 | int64_t size_b = db->estimate_prefix_size("B",""); | |
603 | ASSERT_EQ(size_b, 0); | |
604 | } | |
7c673cae | 605 | |
9f95a23c TL |
606 | fini(); |
607 | } | |
7c673cae | 608 | |
f67539c2 TL |
609 | TEST_P(KVTest, RocksDB_parse_sharding_def) { |
610 | if(string(GetParam()) != "rocksdb") | |
611 | GTEST_SKIP(); | |
612 | ||
613 | bool result; | |
614 | std::vector<RocksDBStore::ColumnFamily> sharding_def; | |
615 | char const* error_position = nullptr; | |
616 | std::string error_msg; | |
617 | ||
618 | std::string_view text_def = "A(10,0-30) B(6)=option1,option2=aaaa C"; | |
619 | result = RocksDBStore::parse_sharding_def(text_def, | |
620 | sharding_def, | |
621 | &error_position, | |
622 | &error_msg); | |
623 | ||
624 | ASSERT_EQ(result, true); | |
625 | ASSERT_EQ(error_position, nullptr); | |
626 | ASSERT_EQ(error_msg, ""); | |
627 | std::cout << text_def << std::endl; | |
628 | if (error_position) std::cout << std::string(error_position - text_def.begin(), ' ') << "^" << error_msg << std::endl; | |
629 | ||
630 | ASSERT_EQ(sharding_def.size(), 3); | |
631 | ASSERT_EQ(sharding_def[0].name, "A"); | |
632 | ASSERT_EQ(sharding_def[0].shard_cnt, 10); | |
633 | ASSERT_EQ(sharding_def[0].hash_l, 0); | |
634 | ASSERT_EQ(sharding_def[0].hash_h, 30); | |
635 | ||
636 | ASSERT_EQ(sharding_def[1].name, "B"); | |
637 | ASSERT_EQ(sharding_def[1].shard_cnt, 6); | |
638 | ASSERT_EQ(sharding_def[1].options, "option1,option2=aaaa"); | |
639 | ASSERT_EQ(sharding_def[2].name, "C"); | |
640 | ASSERT_EQ(sharding_def[2].shard_cnt, 1); | |
641 | ||
642 | ||
643 | text_def = "A(10 B(6)=option C"; | |
644 | result = RocksDBStore::parse_sharding_def(text_def, | |
645 | sharding_def, | |
646 | &error_position, | |
647 | &error_msg); | |
648 | std::cout << text_def << std::endl; | |
649 | if (error_position) | |
650 | std::cout << std::string(error_position - text_def.begin(), ' ') << "^" << error_msg << std::endl; | |
651 | ASSERT_EQ(result, false); | |
652 | ASSERT_NE(error_position, nullptr); | |
653 | ASSERT_NE(error_msg, ""); | |
654 | ||
655 | text_def = "A(10,1) B(6)=option C"; | |
656 | result = RocksDBStore::parse_sharding_def(text_def, | |
657 | sharding_def, | |
658 | &error_position, | |
659 | &error_msg); | |
660 | std::cout << text_def << std::endl; | |
661 | std::cout << std::string(error_position - text_def.begin(), ' ') << "^" << error_msg << std::endl; | |
662 | ASSERT_EQ(result, false); | |
663 | ASSERT_NE(error_position, nullptr); | |
664 | ASSERT_NE(error_msg, ""); | |
665 | } | |
666 | ||
667 | ||
668 | ||
669 | class RocksDBShardingTest : public ::testing::TestWithParam<const char*> { | |
670 | public: | |
671 | boost::scoped_ptr<KeyValueDB> db; | |
672 | ||
673 | RocksDBShardingTest() : db(0) {} | |
674 | ||
675 | string _bl_to_str(bufferlist val) { | |
676 | string str(val.c_str(), val.length()); | |
677 | return str; | |
678 | } | |
679 | ||
680 | void rm_r(string path) { | |
681 | string cmd = string("rm -r ") + path; | |
682 | if (verbose) | |
683 | cout << "==> " << cmd << std::endl; | |
684 | int r = ::system(cmd.c_str()); | |
685 | if (r) { | |
686 | cerr << "failed with exit code " << r | |
687 | << ", continuing anyway" << std::endl; | |
688 | } | |
689 | } | |
690 | ||
691 | void SetUp() override { | |
692 | verbose = getenv("VERBOSE") && strcmp(getenv("VERBOSE"), "1") == 0; | |
693 | ||
694 | int r = ::mkdir("kv_test_temp_dir", 0777); | |
695 | if (r < 0 && errno != EEXIST) { | |
696 | r = -errno; | |
697 | cerr << __func__ << ": unable to create kv_test_temp_dir: " | |
698 | << cpp_strerror(r) << std::endl; | |
699 | return; | |
700 | } | |
701 | db.reset(KeyValueDB::create(g_ceph_context, "rocksdb", | |
702 | "kv_test_temp_dir")); | |
703 | ASSERT_EQ(0, db->init(g_conf()->bluestore_rocksdb_options)); | |
704 | if (verbose) | |
705 | cout << "Creating database with sharding: " << GetParam() << std::endl; | |
706 | ASSERT_EQ(0, db->create_and_open(cout, GetParam())); | |
707 | } | |
708 | void TearDown() override { | |
709 | db.reset(nullptr); | |
710 | rm_r("kv_test_temp_dir"); | |
711 | } | |
712 | ||
713 | /* | |
714 | A - main 0/1/20 | |
715 | B - shard 1/3 x 0/1/20 | |
716 | C - main 0/1/20 | |
717 | D - shard 1/3 x 0/1/20 | |
718 | E - main 0/1/20 | |
719 | */ | |
720 | bool verbose; | |
721 | std::vector<std::string> sharding_defs = { | |
722 | "Betelgeuse D", | |
723 | "Betelgeuse(3) D", | |
724 | "Betelgeuse D(3)", | |
725 | "Betelgeuse(3) D(3)"}; | |
726 | std::vector<std::string> prefixes = {"Ad", "Betelgeuse", "C", "D", "Evade"}; | |
727 | std::vector<std::string> randoms = {"0", "1", "2", "3", "4", "5", | |
728 | "found", "brain", "fully", "pen", "worth", "race", | |
729 | "stand", "nodded", "whenever", "surrounded", "industrial", "skin", | |
730 | "this", "direction", "family", "beginning", "whenever", "held", | |
731 | "metal", "year", "like", "valuable", "softly", "whistle", | |
732 | "perfectly", "broken", "idea", "also", "coffee", "branch", | |
733 | "tongue", "immediately", "bent", "partly", "burn", "include", | |
734 | "certain", "burst", "final", "smoke", "positive", "perfectly" | |
735 | }; | |
736 | int R = randoms.size(); | |
737 | ||
738 | typedef int test_id[6]; | |
739 | void zero(test_id& x) { | |
740 | k = 0; | |
741 | v = 0; | |
742 | for (auto& i:x) | |
743 | i = 0; | |
744 | } | |
745 | bool end(const test_id& x) { | |
746 | return x[5] != 0; | |
747 | } | |
748 | void next(test_id& x) { | |
749 | x[0]++; | |
750 | for (int i = 0; i < 5; i++) { | |
751 | if (x[i] == 3) { | |
752 | x[i] = 0; | |
753 | ++x[i + 1]; | |
754 | } | |
755 | } | |
756 | } | |
757 | ||
758 | std::map<std::string, std::string> data; | |
759 | int k = 0; | |
760 | int v = 0; | |
761 | ||
762 | void generate_data(const test_id& x) { | |
763 | data.clear(); | |
764 | for (int i = 0; i < 5; i++) { | |
765 | if (verbose) | |
766 | std::cout << x[i] << "-"; | |
767 | switch (x[i]) { | |
768 | case 0: | |
769 | break; | |
770 | case 1: | |
771 | data[RocksDBStore::combine_strings(prefixes[i], randoms[k++ % R])] = randoms[v++ % R]; | |
772 | break; | |
773 | case 2: | |
774 | std::string base = randoms[k++ % R]; | |
775 | for (int j = 0; j < 10; j++) { | |
776 | data[RocksDBStore::combine_strings(prefixes[i], base + "." + randoms[k++ % R])] = randoms[v++ % R]; | |
777 | } | |
778 | break; | |
779 | } | |
780 | } | |
781 | } | |
782 | ||
783 | void data_to_db() { | |
784 | KeyValueDB::Transaction t = db->get_transaction(); | |
785 | for (auto &d : data) { | |
786 | bufferlist v1; | |
787 | v1.append(d.second); | |
788 | string prefix; | |
789 | string key; | |
790 | RocksDBStore::split_key(d.first, &prefix, &key); | |
791 | t->set(prefix, key, v1); | |
792 | if (verbose) | |
793 | std::cout << "SET " << prefix << " " << key << std::endl; | |
794 | } | |
795 | ASSERT_EQ(db->submit_transaction_sync(t), 0); | |
796 | } | |
797 | ||
798 | void clear_db() { | |
799 | KeyValueDB::Transaction t = db->get_transaction(); | |
800 | for (auto &d : data) { | |
801 | string prefix; | |
802 | string key; | |
803 | RocksDBStore::split_key(d.first, &prefix, &key); | |
804 | t->rmkey(prefix, key); | |
805 | } | |
806 | ASSERT_EQ(db->submit_transaction_sync(t), 0); | |
807 | //paranoid, check if db empty | |
808 | KeyValueDB::WholeSpaceIterator it = db->get_wholespace_iterator(); | |
809 | ASSERT_EQ(it->seek_to_first(), 0); | |
810 | ASSERT_EQ(it->valid(), false); | |
811 | } | |
812 | }; | |
813 | ||
814 | TEST_P(RocksDBShardingTest, wholespace_next) { | |
815 | test_id X; | |
816 | zero(X); | |
817 | do { | |
818 | generate_data(X); | |
819 | data_to_db(); | |
820 | ||
821 | KeyValueDB::WholeSpaceIterator it = db->get_wholespace_iterator(); | |
822 | //move forward | |
823 | auto dit = data.begin(); | |
824 | int r = it->seek_to_first(); | |
825 | ASSERT_EQ(r, 0); | |
826 | ASSERT_EQ(it->valid(), (dit != data.end())); | |
827 | ||
828 | while (dit != data.end()) { | |
829 | ASSERT_EQ(it->valid(), true); | |
830 | string prefix; | |
831 | string key; | |
832 | RocksDBStore::split_key(dit->first, &prefix, &key); | |
833 | auto raw_key = it->raw_key(); | |
834 | ASSERT_EQ(raw_key.first, prefix); | |
835 | ASSERT_EQ(raw_key.second, key); | |
836 | ASSERT_EQ(it->value().to_str(), dit->second); | |
837 | if (verbose) | |
838 | std::cout << "next " << prefix << " " << key << std::endl; | |
839 | ASSERT_EQ(it->next(), 0); | |
840 | ++dit; | |
841 | } | |
842 | ASSERT_EQ(it->valid(), false); | |
843 | ||
844 | clear_db(); | |
845 | next(X); | |
846 | } while (!end(X)); | |
847 | } | |
848 | ||
849 | TEST_P(RocksDBShardingTest, wholespace_prev) { | |
850 | test_id X; | |
851 | zero(X); | |
852 | do { | |
853 | generate_data(X); | |
854 | data_to_db(); | |
855 | ||
856 | KeyValueDB::WholeSpaceIterator it = db->get_wholespace_iterator(); | |
857 | auto dit = data.rbegin(); | |
858 | int r = it->seek_to_last(); | |
859 | ASSERT_EQ(r, 0); | |
860 | ASSERT_EQ(it->valid(), (dit != data.rend())); | |
861 | ||
862 | while (dit != data.rend()) { | |
863 | ASSERT_EQ(it->valid(), true); | |
864 | string prefix; | |
865 | string key; | |
866 | RocksDBStore::split_key(dit->first, &prefix, &key); | |
867 | auto raw_key = it->raw_key(); | |
868 | ASSERT_EQ(raw_key.first, prefix); | |
869 | ASSERT_EQ(raw_key.second, key); | |
870 | ASSERT_EQ(it->value().to_str(), dit->second); | |
871 | if (verbose) | |
872 | std::cout << "prev " << prefix << " " << key << std::endl; | |
873 | ASSERT_EQ(it->prev(), 0); | |
874 | ++dit; | |
875 | } | |
876 | ASSERT_EQ(it->valid(), false); | |
877 | ||
878 | clear_db(); | |
879 | next(X); | |
880 | } while (!end(X)); | |
881 | } | |
882 | ||
883 | TEST_P(RocksDBShardingTest, wholespace_lower_bound) { | |
884 | test_id X; | |
885 | zero(X); | |
886 | do { | |
887 | generate_data(X); | |
888 | data_to_db(); | |
889 | ||
890 | KeyValueDB::WholeSpaceIterator it = db->get_wholespace_iterator(); | |
891 | auto dit = data.begin(); | |
892 | int r = it->seek_to_first(); | |
893 | ASSERT_EQ(r, 0); | |
894 | ASSERT_EQ(it->valid(), (dit != data.end())); | |
895 | ||
896 | while (dit != data.end()) { | |
897 | ASSERT_EQ(it->valid(), true); | |
898 | string prefix; | |
899 | string key; | |
900 | RocksDBStore::split_key(dit->first, &prefix, &key); | |
901 | KeyValueDB::WholeSpaceIterator it1 = db->get_wholespace_iterator(); | |
902 | ASSERT_EQ(it1->lower_bound(prefix, key), 0); | |
903 | ASSERT_EQ(it1->valid(), true); | |
904 | auto raw_key = it1->raw_key(); | |
905 | ASSERT_EQ(raw_key.first, prefix); | |
906 | ASSERT_EQ(raw_key.second, key); | |
907 | if (verbose) | |
908 | std::cout << "lower_bound " << prefix << " " << key << std::endl; | |
909 | ASSERT_EQ(it->next(), 0); | |
910 | ++dit; | |
911 | } | |
912 | ASSERT_EQ(it->valid(), false); | |
913 | ||
914 | clear_db(); | |
915 | next(X); | |
916 | } while (!end(X)); | |
917 | } | |
918 | ||
919 | TEST_P(RocksDBShardingTest, wholespace_upper_bound) { | |
920 | test_id X; | |
921 | zero(X); | |
922 | do { | |
923 | generate_data(X); | |
924 | data_to_db(); | |
925 | ||
926 | KeyValueDB::WholeSpaceIterator it = db->get_wholespace_iterator(); | |
927 | auto dit = data.begin(); | |
928 | int r = it->seek_to_first(); | |
929 | ASSERT_EQ(r, 0); | |
930 | ASSERT_EQ(it->valid(), (dit != data.end())); | |
931 | ||
932 | while (dit != data.end()) { | |
933 | ASSERT_EQ(it->valid(), true); | |
934 | string prefix; | |
935 | string key; | |
936 | string key_minus_1; | |
937 | RocksDBStore::split_key(dit->first, &prefix, &key); | |
938 | //decrement key minimally | |
939 | key_minus_1 = key.substr(0, key.length() - 1) + std::string(1, key[key.length() - 1] - 1); | |
940 | KeyValueDB::WholeSpaceIterator it1 = db->get_wholespace_iterator(); | |
941 | ASSERT_EQ(it1->upper_bound(prefix, key_minus_1), 0); | |
942 | ASSERT_EQ(it1->valid(), true); | |
943 | auto raw_key = it1->raw_key(); | |
944 | ASSERT_EQ(raw_key.first, prefix); | |
945 | ASSERT_EQ(raw_key.second, key); | |
946 | if (verbose) | |
947 | std::cout << "upper_bound " << prefix << " " << key_minus_1 << std::endl; | |
948 | ASSERT_EQ(it->next(), 0); | |
949 | ++dit; | |
950 | } | |
951 | ASSERT_EQ(it->valid(), false); | |
952 | ||
953 | clear_db(); | |
954 | next(X); | |
955 | } while (!end(X)); | |
956 | } | |
957 | ||
958 | TEST_P(RocksDBShardingTest, wholespace_lookup_limits) { | |
959 | test_id X; | |
960 | zero(X); | |
961 | do { | |
962 | generate_data(X); | |
963 | data_to_db(); | |
964 | ||
965 | //lookup before first | |
966 | if (data.size() > 0) { | |
967 | auto dit = data.begin(); | |
968 | string prefix; | |
969 | string key; | |
970 | RocksDBStore::split_key(dit->first, &prefix, &key); | |
971 | KeyValueDB::WholeSpaceIterator it1 = db->get_wholespace_iterator(); | |
972 | ASSERT_EQ(it1->lower_bound(" ", " "), 0); | |
973 | ASSERT_EQ(it1->valid(), true); | |
974 | auto raw_key = it1->raw_key(); | |
975 | ASSERT_EQ(raw_key.first, prefix); | |
976 | ASSERT_EQ(raw_key.second, key); | |
977 | } | |
978 | //lookup after last | |
979 | KeyValueDB::WholeSpaceIterator it1 = db->get_wholespace_iterator(); | |
980 | ASSERT_EQ(it1->lower_bound("~", "~"), 0); | |
981 | ASSERT_EQ(it1->valid(), false); | |
982 | ||
983 | clear_db(); | |
984 | next(X); | |
985 | } while (!end(X)); | |
986 | } | |
987 | ||
988 | ||
989 | ||
990 | class RocksDBResharding : public ::testing::Test { | |
991 | public: | |
992 | boost::scoped_ptr<RocksDBStore> db; | |
993 | ||
994 | RocksDBResharding() : db(0) {} | |
995 | ||
996 | string _bl_to_str(bufferlist val) { | |
997 | string str(val.c_str(), val.length()); | |
998 | return str; | |
999 | } | |
1000 | ||
1001 | void rm_r(string path) { | |
1002 | string cmd = string("rm -r ") + path; | |
1003 | if (verbose) | |
1004 | cout << "==> " << cmd << std::endl; | |
1005 | int r = ::system(cmd.c_str()); | |
1006 | if (r) { | |
1007 | cerr << "failed with exit code " << r | |
1008 | << ", continuing anyway" << std::endl; | |
1009 | } | |
1010 | } | |
1011 | ||
1012 | void SetUp() override { | |
1013 | verbose = getenv("VERBOSE") && strcmp(getenv("VERBOSE"), "1") == 0; | |
1014 | ||
1015 | int r = ::mkdir("kv_test_temp_dir", 0777); | |
1016 | if (r < 0 && errno != EEXIST) { | |
1017 | r = -errno; | |
1018 | cerr << __func__ << ": unable to create kv_test_temp_dir: " | |
1019 | << cpp_strerror(r) << std::endl; | |
1020 | return; | |
1021 | } | |
1022 | ||
1023 | KeyValueDB* db_kv = KeyValueDB::create(g_ceph_context, "rocksdb", | |
1024 | "kv_test_temp_dir"); | |
1025 | RocksDBStore* db_rocks = dynamic_cast<RocksDBStore*>(db_kv); | |
1026 | ceph_assert(db_rocks); | |
1027 | db.reset(db_rocks); | |
1028 | ASSERT_EQ(0, db->init(g_conf()->bluestore_rocksdb_options)); | |
1029 | } | |
1030 | void TearDown() override { | |
1031 | db.reset(nullptr); | |
1032 | rm_r("kv_test_temp_dir"); | |
1033 | } | |
1034 | ||
1035 | bool verbose; | |
1036 | std::vector<std::string> prefixes = {"Ad", "Betelgeuse", "C", "D", "Evade"}; | |
1037 | std::vector<std::string> randoms = {"0", "1", "2", "3", "4", "5", | |
1038 | "found", "brain", "fully", "pen", "worth", "race", | |
1039 | "stand", "nodded", "whenever", "surrounded", "industrial", "skin", | |
1040 | "this", "direction", "family", "beginning", "whenever", "held", | |
1041 | "metal", "year", "like", "valuable", "softly", "whistle", | |
1042 | "perfectly", "broken", "idea", "also", "coffee", "branch", | |
1043 | "tongue", "immediately", "bent", "partly", "burn", "include", | |
1044 | "certain", "burst", "final", "smoke", "positive", "perfectly" | |
1045 | }; | |
1046 | int R = randoms.size(); | |
1047 | int k = 0; | |
1048 | std::map<std::string, std::string> data; | |
1049 | ||
1050 | void generate_data() { | |
1051 | data.clear(); | |
1052 | for (size_t p = 0; p < prefixes.size(); p++) { | |
1053 | size_t elem_count = 1 << (( p * 3 ) + 3); | |
1054 | for (size_t i = 0; i < elem_count; i++) { | |
1055 | std::string key; | |
1056 | for (int x = 0; x < 5; x++) { | |
1057 | key = key + randoms[rand() % R]; | |
1058 | } | |
1059 | std::string value; | |
1060 | for (int x = 0; x < 3; x++) { | |
1061 | value = value + randoms[rand() % R]; | |
1062 | } | |
1063 | data[RocksDBStore::combine_strings(prefixes[p], key)] = value; | |
1064 | } | |
1065 | } | |
1066 | } | |
1067 | ||
1068 | void data_to_db() { | |
1069 | KeyValueDB::Transaction t = db->get_transaction(); | |
1070 | size_t i = 0; | |
1071 | for (auto& d: data) { | |
1072 | bufferlist v1; | |
1073 | v1.append(d.second); | |
1074 | string prefix; | |
1075 | string key; | |
1076 | RocksDBStore::split_key(d.first, &prefix, &key); | |
1077 | t->set(prefix, key, v1); | |
1078 | if (verbose) | |
1079 | std::cout << "SET " << prefix << " " << key << std::endl; | |
1080 | i++; | |
1081 | if ((i % 1000) == 0) { | |
1082 | ASSERT_EQ(db->submit_transaction_sync(t), 0); | |
1083 | t.reset(); | |
1084 | if (verbose) | |
1085 | std::cout << "writing key to DB" << std::endl; | |
1086 | t = db->get_transaction(); | |
1087 | } | |
1088 | } | |
1089 | if (verbose) | |
1090 | std::cout << "writing keys to DB" << std::endl; | |
1091 | ASSERT_EQ(db->submit_transaction_sync(t), 0); | |
1092 | } | |
1093 | ||
1094 | void clear_db() { | |
1095 | KeyValueDB::Transaction t = db->get_transaction(); | |
1096 | for (auto &d : data) { | |
1097 | string prefix; | |
1098 | string key; | |
1099 | RocksDBStore::split_key(d.first, &prefix, &key); | |
1100 | t->rmkey(prefix, key); | |
1101 | } | |
1102 | ASSERT_EQ(db->submit_transaction_sync(t), 0); | |
1103 | //paranoid, check if db empty | |
1104 | KeyValueDB::WholeSpaceIterator it = db->get_wholespace_iterator(); | |
1105 | ASSERT_EQ(it->seek_to_first(), 0); | |
1106 | ASSERT_EQ(it->valid(), false); | |
1107 | } | |
1108 | ||
1109 | void check_db() { | |
1110 | KeyValueDB::WholeSpaceIterator it = db->get_wholespace_iterator(); | |
1111 | //move forward | |
1112 | auto dit = data.begin(); | |
1113 | int r = it->seek_to_first(); | |
1114 | ASSERT_EQ(r, 0); | |
1115 | ASSERT_EQ(it->valid(), (dit != data.end())); | |
1116 | ||
1117 | while (dit != data.end()) { | |
1118 | ASSERT_EQ(it->valid(), true); | |
1119 | string prefix; | |
1120 | string key; | |
1121 | RocksDBStore::split_key(dit->first, &prefix, &key); | |
1122 | auto raw_key = it->raw_key(); | |
1123 | ASSERT_EQ(raw_key.first, prefix); | |
1124 | ASSERT_EQ(raw_key.second, key); | |
1125 | ASSERT_EQ(it->value().to_str(), dit->second); | |
1126 | if (verbose) | |
1127 | std::cout << "next " << prefix << " " << key << std::endl; | |
1128 | ASSERT_EQ(it->next(), 0); | |
1129 | ++dit; | |
1130 | } | |
1131 | ASSERT_EQ(it->valid(), false); | |
1132 | } | |
1133 | }; | |
1134 | ||
1135 | TEST_F(RocksDBResharding, basic) { | |
1136 | ASSERT_EQ(0, db->create_and_open(cout, "")); | |
1137 | generate_data(); | |
1138 | data_to_db(); | |
1139 | check_db(); | |
1140 | db->close(); | |
1141 | ASSERT_EQ(db->reshard("Evade(4)"), 0); | |
1142 | ASSERT_EQ(db->open(cout), 0); | |
1143 | check_db(); | |
1144 | db->close(); | |
1145 | } | |
1146 | ||
1147 | TEST_F(RocksDBResharding, all_to_shards) { | |
1148 | ASSERT_EQ(0, db->create_and_open(cout, "")); | |
1149 | generate_data(); | |
1150 | data_to_db(); | |
1151 | check_db(); | |
1152 | db->close(); | |
1153 | ASSERT_EQ(db->reshard("Ad(1) Betelgeuse(1) C(1) D(1) Evade(1)"), 0); | |
1154 | ASSERT_EQ(db->open(cout), 0); | |
1155 | check_db(); | |
1156 | db->close(); | |
1157 | } | |
1158 | ||
1159 | TEST_F(RocksDBResharding, all_to_shards_and_back_again) { | |
1160 | ASSERT_EQ(0, db->create_and_open(cout, "")); | |
1161 | generate_data(); | |
1162 | data_to_db(); | |
1163 | check_db(); | |
1164 | db->close(); | |
1165 | ASSERT_EQ(db->reshard("Ad(1) Betelgeuse(1) C(1) D(1) Evade(1)"), 0); | |
1166 | ASSERT_EQ(db->open(cout), 0); | |
1167 | check_db(); | |
1168 | db->close(); | |
1169 | ASSERT_EQ(db->reshard(""), 0); | |
1170 | ASSERT_EQ(db->open(cout), 0); | |
1171 | check_db(); | |
1172 | db->close(); | |
1173 | } | |
1174 | ||
1175 | TEST_F(RocksDBResharding, resume_interrupted_at_batch) { | |
1176 | ASSERT_EQ(0, db->create_and_open(cout, "")); | |
1177 | generate_data(); | |
1178 | data_to_db(); | |
1179 | check_db(); | |
1180 | db->close(); | |
1181 | RocksDBStore::resharding_ctrl ctrl; | |
1182 | ctrl.unittest_fail_after_first_batch = true; | |
1183 | ASSERT_EQ(db->reshard("Evade(4)", &ctrl), -1000); | |
1184 | ASSERT_NE(db->open(cout), 0); | |
1185 | ASSERT_EQ(db->reshard("Evade(4)"), 0); | |
1186 | ASSERT_EQ(db->open(cout), 0); | |
1187 | check_db(); | |
1188 | db->close(); | |
1189 | } | |
1190 | ||
1191 | TEST_F(RocksDBResharding, resume_interrupted_at_column) { | |
1192 | ASSERT_EQ(0, db->create_and_open(cout, "")); | |
1193 | generate_data(); | |
1194 | data_to_db(); | |
1195 | check_db(); | |
1196 | db->close(); | |
1197 | RocksDBStore::resharding_ctrl ctrl; | |
1198 | ctrl.unittest_fail_after_processing_column = true; | |
1199 | ASSERT_EQ(db->reshard("Evade(4)", &ctrl), -1001); | |
1200 | ASSERT_NE(db->open(cout), 0); | |
1201 | ASSERT_EQ(db->reshard("Evade(4)"), 0); | |
1202 | ASSERT_EQ(db->open(cout), 0); | |
1203 | check_db(); | |
1204 | db->close(); | |
1205 | } | |
1206 | ||
1207 | TEST_F(RocksDBResharding, resume_interrupted_before_commit) { | |
1208 | ASSERT_EQ(0, db->create_and_open(cout, "")); | |
1209 | generate_data(); | |
1210 | data_to_db(); | |
1211 | check_db(); | |
1212 | db->close(); | |
1213 | RocksDBStore::resharding_ctrl ctrl; | |
1214 | ctrl.unittest_fail_after_successful_processing = true; | |
1215 | ASSERT_EQ(db->reshard("Evade(4)", &ctrl), -1002); | |
1216 | ASSERT_NE(db->open(cout), 0); | |
1217 | ASSERT_EQ(db->reshard("Evade(4)"), 0); | |
1218 | ASSERT_EQ(db->open(cout), 0); | |
1219 | check_db(); | |
1220 | db->close(); | |
1221 | } | |
1222 | ||
1223 | TEST_F(RocksDBResharding, prevent_incomplete_hash_change) { | |
1224 | ASSERT_EQ(0, db->create_and_open(cout, "Evade(4,0-3)")); | |
1225 | generate_data(); | |
1226 | data_to_db(); | |
1227 | check_db(); | |
1228 | db->close(); | |
1229 | RocksDBStore::resharding_ctrl ctrl; | |
1230 | ctrl.unittest_fail_after_successful_processing = true; | |
1231 | ASSERT_EQ(db->reshard("Evade(4,0-8)", &ctrl), -1002); | |
1232 | ASSERT_NE(db->open(cout), 0); | |
1233 | ASSERT_EQ(db->reshard("Evade(4,0-8)"), 0); | |
1234 | ASSERT_EQ(db->open(cout), 0); | |
1235 | check_db(); | |
1236 | db->close(); | |
1237 | } | |
1238 | ||
1239 | TEST_F(RocksDBResharding, change_reshard) { | |
1240 | ASSERT_EQ(0, db->create_and_open(cout, "Ad(4)")); | |
1241 | generate_data(); | |
1242 | data_to_db(); | |
1243 | check_db(); | |
1244 | db->close(); | |
1245 | { | |
1246 | RocksDBStore::resharding_ctrl ctrl; | |
1247 | ctrl.unittest_fail_after_first_batch = true; | |
1248 | ASSERT_EQ(db->reshard("C(5) D(3)", &ctrl), -1000); | |
1249 | } | |
1250 | { | |
1251 | RocksDBStore::resharding_ctrl ctrl; | |
1252 | ASSERT_NE(db->open(cout), 0); | |
1253 | ctrl.unittest_fail_after_first_batch = false; | |
1254 | ctrl.unittest_fail_after_processing_column = true; | |
1255 | ASSERT_EQ(db->reshard("C(5) Evade(2)", &ctrl), -1001); | |
1256 | } | |
1257 | { | |
1258 | RocksDBStore::resharding_ctrl ctrl; | |
1259 | ASSERT_NE(db->open(cout), 0); | |
1260 | ctrl.unittest_fail_after_processing_column = false; | |
1261 | ctrl.unittest_fail_after_successful_processing = true; | |
1262 | ASSERT_EQ(db->reshard("Evade(2) D(3)", &ctrl), -1002); | |
1263 | } | |
1264 | { | |
1265 | ASSERT_NE(db->open(cout), 0); | |
1266 | ASSERT_EQ(db->reshard("Ad(1) Evade(5)"), 0); | |
1267 | } | |
1268 | { | |
1269 | ASSERT_EQ(db->open(cout), 0); | |
1270 | check_db(); | |
1271 | db->close(); | |
1272 | } | |
1273 | } | |
1274 | ||
1275 | ||
9f95a23c TL |
1276 | INSTANTIATE_TEST_SUITE_P( |
1277 | KeyValueDB, | |
1278 | KVTest, | |
1279 | ::testing::Values("leveldb", "rocksdb", "memdb")); | |
7c673cae | 1280 | |
f67539c2 TL |
1281 | INSTANTIATE_TEST_SUITE_P( |
1282 | KeyValueDB, | |
1283 | RocksDBShardingTest, | |
1284 | ::testing::Values("Betelgeuse D", | |
1285 | "Betelgeuse(3) D", | |
1286 | "Betelgeuse D(3)", | |
1287 | "Betelgeuse(3) D(3)")); | |
1288 | ||
7c673cae FG |
1289 | int main(int argc, char **argv) { |
1290 | vector<const char*> args; | |
1291 | argv_to_vec(argc, (const char **)argv, args); | |
7c673cae FG |
1292 | |
1293 | auto cct = global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, | |
11fdf7f2 TL |
1294 | CODE_ENVIRONMENT_UTILITY, |
1295 | CINIT_FLAG_NO_DEFAULT_CONFIG_FILE); | |
7c673cae | 1296 | common_init_finish(g_ceph_context); |
11fdf7f2 | 1297 | g_ceph_context->_conf.set_val( |
7c673cae FG |
1298 | "enable_experimental_unrecoverable_data_corrupting_features", |
1299 | "rocksdb, memdb"); | |
11fdf7f2 | 1300 | g_ceph_context->_conf.apply_changes(nullptr); |
7c673cae FG |
1301 | |
1302 | ::testing::InitGoogleTest(&argc, argv); | |
1303 | return RUN_ALL_TESTS(); | |
1304 | } |