]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- |
2 | // vim: ts=8 sw=2 smarttab | |
3 | /* | |
4 | * Ceph - scalable distributed file system | |
5 | * | |
6 | * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net> | |
7 | * | |
8 | * This is free software; you can redistribute it and/or | |
9 | * modify it under the terms of the GNU Lesser General Public | |
10 | * License version 2.1, as published by the Free Software | |
11 | * Foundation. See file COPYING. | |
12 | * | |
13 | */ | |
14 | ||
15 | #include <stdio.h> | |
16 | #include <string.h> | |
17 | #include <iostream> | |
18 | #include <time.h> | |
19 | #include <sys/mount.h> | |
20 | #include "kv/KeyValueDB.h" | |
f67539c2 | 21 | #include "kv/RocksDBStore.h" |
7c673cae FG |
22 | #include "include/Context.h" |
23 | #include "common/ceph_argparse.h" | |
24 | #include "global/global_init.h" | |
7c673cae FG |
25 | #include "common/Cond.h" |
26 | #include "common/errno.h" | |
27 | #include "include/stringify.h" | |
28 | #include <gtest/gtest.h> | |
29 | ||
20effc67 TL |
30 | using namespace std; |
31 | ||
7c673cae FG |
32 | class KVTest : public ::testing::TestWithParam<const char*> { |
33 | public: | |
34 | boost::scoped_ptr<KeyValueDB> db; | |
35 | ||
36 | KVTest() : db(0) {} | |
37 | ||
11fdf7f2 TL |
38 | string _bl_to_str(bufferlist val) { |
39 | string str(val.c_str(), val.length()); | |
40 | return str; | |
41 | } | |
42 | ||
7c673cae FG |
43 | void rm_r(string path) { |
44 | string cmd = string("rm -r ") + path; | |
45 | cout << "==> " << cmd << std::endl; | |
46 | int r = ::system(cmd.c_str()); | |
47 | if (r) { | |
48 | cerr << "failed with exit code " << r | |
49 | << ", continuing anyway" << std::endl; | |
50 | } | |
51 | } | |
52 | ||
53 | void init() { | |
54 | cout << "Creating " << string(GetParam()) << "\n"; | |
55 | db.reset(KeyValueDB::create(g_ceph_context, string(GetParam()), | |
56 | "kv_test_temp_dir")); | |
57 | } | |
58 | void fini() { | |
59 | db.reset(NULL); | |
60 | } | |
61 | ||
62 | void SetUp() override { | |
63 | int r = ::mkdir("kv_test_temp_dir", 0777); | |
64 | if (r < 0 && errno != EEXIST) { | |
65 | r = -errno; | |
66 | cerr << __func__ << ": unable to create kv_test_temp_dir: " | |
67 | << cpp_strerror(r) << std::endl; | |
68 | return; | |
69 | } | |
70 | init(); | |
71 | } | |
72 | void TearDown() override { | |
73 | fini(); | |
74 | rm_r("kv_test_temp_dir"); | |
75 | } | |
76 | }; | |
77 | ||
78 | TEST_P(KVTest, OpenClose) { | |
79 | ASSERT_EQ(0, db->create_and_open(cout)); | |
f67539c2 TL |
80 | db->close(); |
81 | db->open(cout); | |
7c673cae FG |
82 | fini(); |
83 | } | |
84 | ||
85 | TEST_P(KVTest, OpenCloseReopenClose) { | |
86 | ASSERT_EQ(0, db->create_and_open(cout)); | |
87 | fini(); | |
88 | init(); | |
89 | ASSERT_EQ(0, db->open(cout)); | |
90 | fini(); | |
91 | } | |
92 | ||
93 | /* | |
94 | * Basic write and read test case in same database session. | |
95 | */ | |
96 | TEST_P(KVTest, OpenWriteRead) { | |
97 | ASSERT_EQ(0, db->create_and_open(cout)); | |
98 | { | |
99 | KeyValueDB::Transaction t = db->get_transaction(); | |
100 | bufferlist value; | |
101 | value.append("value"); | |
102 | t->set("prefix", "key", value); | |
103 | value.clear(); | |
104 | value.append("value2"); | |
105 | t->set("prefix", "key2", value); | |
106 | value.clear(); | |
107 | value.append("value3"); | |
108 | t->set("prefix", "key3", value); | |
109 | db->submit_transaction_sync(t); | |
110 | ||
111 | bufferlist v1, v2; | |
112 | ASSERT_EQ(0, db->get("prefix", "key", &v1)); | |
113 | ASSERT_EQ(v1.length(), 5u); | |
114 | (v1.c_str())[v1.length()] = 0x0; | |
115 | ASSERT_EQ(std::string(v1.c_str()), std::string("value")); | |
116 | ASSERT_EQ(0, db->get("prefix", "key2", &v2)); | |
117 | ASSERT_EQ(v2.length(), 6u); | |
118 | (v2.c_str())[v2.length()] = 0x0; | |
119 | ASSERT_EQ(std::string(v2.c_str()), std::string("value2")); | |
120 | } | |
121 | fini(); | |
122 | } | |
123 | ||
124 | TEST_P(KVTest, PutReopen) { | |
125 | ASSERT_EQ(0, db->create_and_open(cout)); | |
126 | { | |
127 | KeyValueDB::Transaction t = db->get_transaction(); | |
128 | bufferlist value; | |
129 | value.append("value"); | |
130 | t->set("prefix", "key", value); | |
131 | t->set("prefix", "key2", value); | |
132 | t->set("prefix", "key3", value); | |
133 | db->submit_transaction_sync(t); | |
134 | } | |
135 | fini(); | |
136 | ||
137 | init(); | |
138 | ASSERT_EQ(0, db->open(cout)); | |
139 | { | |
140 | bufferlist v1, v2; | |
141 | ASSERT_EQ(0, db->get("prefix", "key", &v1)); | |
142 | ASSERT_EQ(v1.length(), 5u); | |
143 | ASSERT_EQ(0, db->get("prefix", "key2", &v2)); | |
144 | ASSERT_EQ(v2.length(), 5u); | |
145 | } | |
146 | { | |
147 | KeyValueDB::Transaction t = db->get_transaction(); | |
148 | t->rmkey("prefix", "key"); | |
149 | t->rmkey("prefix", "key3"); | |
150 | db->submit_transaction_sync(t); | |
151 | } | |
152 | fini(); | |
153 | ||
154 | init(); | |
155 | ASSERT_EQ(0, db->open(cout)); | |
156 | { | |
157 | bufferlist v1, v2, v3; | |
158 | ASSERT_EQ(-ENOENT, db->get("prefix", "key", &v1)); | |
159 | ASSERT_EQ(0, db->get("prefix", "key2", &v2)); | |
160 | ASSERT_EQ(v2.length(), 5u); | |
161 | ASSERT_EQ(-ENOENT, db->get("prefix", "key3", &v3)); | |
162 | } | |
163 | fini(); | |
164 | } | |
165 | ||
166 | TEST_P(KVTest, BenchCommit) { | |
167 | int n = 1024; | |
168 | ASSERT_EQ(0, db->create_and_open(cout)); | |
169 | utime_t start = ceph_clock_now(); | |
170 | { | |
171 | cout << "priming" << std::endl; | |
172 | // prime | |
173 | bufferlist big; | |
174 | bufferptr bp(1048576); | |
175 | bp.zero(); | |
176 | big.append(bp); | |
177 | for (int i=0; i<30; ++i) { | |
178 | KeyValueDB::Transaction t = db->get_transaction(); | |
179 | t->set("prefix", "big" + stringify(i), big); | |
180 | db->submit_transaction_sync(t); | |
181 | } | |
182 | } | |
183 | cout << "now doing small writes" << std::endl; | |
184 | bufferlist data; | |
185 | bufferptr bp(1024); | |
186 | bp.zero(); | |
187 | data.append(bp); | |
188 | for (int i=0; i<n; ++i) { | |
189 | KeyValueDB::Transaction t = db->get_transaction(); | |
190 | t->set("prefix", "key" + stringify(i), data); | |
191 | db->submit_transaction_sync(t); | |
192 | } | |
193 | utime_t end = ceph_clock_now(); | |
194 | utime_t dur = end - start; | |
195 | cout << n << " commits in " << dur << ", avg latency " << (dur / (double)n) | |
196 | << std::endl; | |
197 | fini(); | |
198 | } | |
199 | ||
200 | struct AppendMOP : public KeyValueDB::MergeOperator { | |
201 | void merge_nonexistent( | |
202 | const char *rdata, size_t rlen, std::string *new_value) override { | |
203 | *new_value = "?" + std::string(rdata, rlen); | |
204 | } | |
205 | void merge( | |
206 | const char *ldata, size_t llen, | |
207 | const char *rdata, size_t rlen, | |
208 | std::string *new_value) override { | |
7c673cae FG |
209 | *new_value = std::string(ldata, llen) + std::string(rdata, rlen); |
210 | } | |
211 | // We use each operator name and each prefix to construct the | |
212 | // overall RocksDB operator name for consistency check at open time. | |
91327a77 | 213 | const char *name() const override { |
7c673cae FG |
214 | return "Append"; |
215 | } | |
216 | }; | |
217 | ||
218 | string tostr(bufferlist& b) { | |
219 | return string(b.c_str(),b.length()); | |
220 | } | |
221 | ||
222 | TEST_P(KVTest, Merge) { | |
223 | shared_ptr<KeyValueDB::MergeOperator> p(new AppendMOP); | |
224 | int r = db->set_merge_operator("A",p); | |
225 | if (r < 0) | |
226 | return; // No merge operators for this database type | |
227 | ASSERT_EQ(0, db->create_and_open(cout)); | |
228 | { | |
229 | KeyValueDB::Transaction t = db->get_transaction(); | |
230 | bufferlist v1, v2, v3; | |
231 | v1.append(string("1")); | |
232 | v2.append(string("2")); | |
233 | v3.append(string("3")); | |
234 | t->set("P", "K1", v1); | |
235 | t->set("A", "A1", v2); | |
236 | t->rmkey("A", "A2"); | |
237 | t->merge("A", "A2", v3); | |
238 | db->submit_transaction_sync(t); | |
239 | } | |
240 | { | |
241 | bufferlist v1, v2, v3; | |
242 | ASSERT_EQ(0, db->get("P", "K1", &v1)); | |
243 | ASSERT_EQ(tostr(v1), "1"); | |
244 | ASSERT_EQ(0, db->get("A", "A1", &v2)); | |
245 | ASSERT_EQ(tostr(v2), "2"); | |
246 | ASSERT_EQ(0, db->get("A", "A2", &v3)); | |
247 | ASSERT_EQ(tostr(v3), "?3"); | |
248 | } | |
249 | { | |
250 | KeyValueDB::Transaction t = db->get_transaction(); | |
251 | bufferlist v1; | |
252 | v1.append(string("1")); | |
253 | t->merge("A", "A2", v1); | |
254 | db->submit_transaction_sync(t); | |
255 | } | |
256 | { | |
257 | bufferlist v; | |
258 | ASSERT_EQ(0, db->get("A", "A2", &v)); | |
259 | ASSERT_EQ(tostr(v), "?31"); | |
260 | } | |
261 | fini(); | |
262 | } | |
263 | ||
264 | TEST_P(KVTest, RMRange) { | |
265 | ASSERT_EQ(0, db->create_and_open(cout)); | |
266 | bufferlist value; | |
267 | value.append("value"); | |
268 | { | |
269 | KeyValueDB::Transaction t = db->get_transaction(); | |
270 | t->set("prefix", "key1", value); | |
271 | t->set("prefix", "key2", value); | |
272 | t->set("prefix", "key3", value); | |
273 | t->set("prefix", "key4", value); | |
274 | t->set("prefix", "key45", value); | |
275 | t->set("prefix", "key5", value); | |
276 | t->set("prefix", "key6", value); | |
277 | db->submit_transaction_sync(t); | |
278 | } | |
279 | ||
280 | { | |
281 | KeyValueDB::Transaction t = db->get_transaction(); | |
282 | t->set("prefix", "key7", value); | |
283 | t->set("prefix", "key8", value); | |
284 | t->rm_range_keys("prefix", "key2", "key7"); | |
285 | db->submit_transaction_sync(t); | |
286 | bufferlist v1, v2; | |
287 | ASSERT_EQ(0, db->get("prefix", "key1", &v1)); | |
288 | v1.clear(); | |
289 | ASSERT_EQ(-ENOENT, db->get("prefix", "key45", &v1)); | |
290 | ASSERT_EQ(0, db->get("prefix", "key8", &v1)); | |
291 | v1.clear(); | |
292 | ASSERT_EQ(-ENOENT, db->get("prefix", "key2", &v1)); | |
293 | ASSERT_EQ(0, db->get("prefix", "key7", &v2)); | |
294 | } | |
295 | ||
296 | { | |
297 | KeyValueDB::Transaction t = db->get_transaction(); | |
298 | t->rm_range_keys("prefix", "key", "key"); | |
299 | db->submit_transaction_sync(t); | |
300 | bufferlist v1, v2; | |
301 | ASSERT_EQ(0, db->get("prefix", "key1", &v1)); | |
302 | ASSERT_EQ(0, db->get("prefix", "key8", &v2)); | |
303 | } | |
304 | ||
305 | { | |
306 | KeyValueDB::Transaction t = db->get_transaction(); | |
307 | t->rm_range_keys("prefix", "key-", "key~"); | |
308 | db->submit_transaction_sync(t); | |
309 | bufferlist v1, v2; | |
310 | ASSERT_EQ(-ENOENT, db->get("prefix", "key1", &v1)); | |
311 | ASSERT_EQ(-ENOENT, db->get("prefix", "key8", &v2)); | |
312 | } | |
313 | ||
314 | fini(); | |
315 | } | |
316 | ||
f67539c2 TL |
317 | TEST_P(KVTest, ShardingRMRange) { |
318 | if(string(GetParam()) != "rocksdb") | |
319 | return; | |
320 | std::string cfs("O(7)="); | |
321 | ASSERT_EQ(0, db->create_and_open(cout, cfs)); | |
322 | { | |
323 | KeyValueDB::Transaction t = db->get_transaction(); | |
324 | for (size_t i = 0; i < 1000; i++) { | |
325 | bufferlist value; | |
326 | char* a; | |
327 | ASSERT_EQ(asprintf(&a, "key%3.3ld", i), 6); | |
328 | value.append(a); | |
329 | t->set("O", a, value); | |
330 | free(a); | |
331 | } | |
332 | db->submit_transaction_sync(t); | |
333 | } | |
334 | ||
335 | { | |
336 | KeyValueDB::Transaction t = db->get_transaction(); | |
337 | t->rm_range_keys("O", "key277", "key467"); | |
338 | db->submit_transaction_sync(t); | |
339 | } | |
340 | ||
341 | for (size_t i = 0; i < 1000; i++) { | |
342 | char* key; | |
343 | ASSERT_EQ(asprintf(&key, "key%3.3ld", i), 6); | |
344 | bufferlist value; | |
345 | int r = db->get("O", key, &value); | |
346 | ASSERT_EQ(r, (i >= 277 && i < 467 ? -ENOENT : 0)); | |
347 | free(key); | |
348 | } | |
349 | ||
350 | fini(); | |
351 | } | |
352 | ||
353 | ||
11fdf7f2 TL |
354 | TEST_P(KVTest, RocksDBColumnFamilyTest) { |
355 | if(string(GetParam()) != "rocksdb") | |
356 | return; | |
357 | ||
f67539c2 | 358 | std::string cfs("cf1 cf2"); |
11fdf7f2 TL |
359 | ASSERT_EQ(0, db->init(g_conf()->bluestore_rocksdb_options)); |
360 | cout << "creating two column families and opening them" << std::endl; | |
361 | ASSERT_EQ(0, db->create_and_open(cout, cfs)); | |
362 | { | |
363 | KeyValueDB::Transaction t = db->get_transaction(); | |
364 | bufferlist value; | |
365 | value.append("value"); | |
366 | cout << "write a transaction includes three keys in different CFs" << std::endl; | |
367 | t->set("prefix", "key", value); | |
368 | t->set("cf1", "key", value); | |
369 | t->set("cf2", "key2", value); | |
370 | ASSERT_EQ(0, db->submit_transaction_sync(t)); | |
371 | } | |
372 | fini(); | |
373 | ||
374 | init(); | |
375 | ASSERT_EQ(0, db->open(cout, cfs)); | |
376 | { | |
377 | bufferlist v1, v2, v3; | |
378 | cout << "reopen db and read those keys" << std::endl; | |
379 | ASSERT_EQ(0, db->get("prefix", "key", &v1)); | |
380 | ASSERT_EQ(0, _bl_to_str(v1) != "value"); | |
381 | ASSERT_EQ(0, db->get("cf1", "key", &v2)); | |
382 | ASSERT_EQ(0, _bl_to_str(v2) != "value"); | |
383 | ASSERT_EQ(0, db->get("cf2", "key2", &v3)); | |
384 | ASSERT_EQ(0, _bl_to_str(v2) != "value"); | |
385 | } | |
386 | { | |
387 | cout << "delete two keys in CFs" << std::endl; | |
388 | KeyValueDB::Transaction t = db->get_transaction(); | |
389 | t->rmkey("prefix", "key"); | |
390 | t->rmkey("cf2", "key2"); | |
391 | ASSERT_EQ(0, db->submit_transaction_sync(t)); | |
392 | } | |
393 | fini(); | |
394 | ||
395 | init(); | |
396 | ASSERT_EQ(0, db->open(cout, cfs)); | |
397 | { | |
398 | cout << "reopen db and read keys again." << std::endl; | |
399 | bufferlist v1, v2, v3; | |
400 | ASSERT_EQ(-ENOENT, db->get("prefix", "key", &v1)); | |
401 | ASSERT_EQ(0, db->get("cf1", "key", &v2)); | |
402 | ASSERT_EQ(0, _bl_to_str(v2) != "value"); | |
403 | ASSERT_EQ(-ENOENT, db->get("cf2", "key2", &v3)); | |
404 | } | |
405 | fini(); | |
406 | } | |
407 | ||
408 | TEST_P(KVTest, RocksDBIteratorTest) { | |
409 | if(string(GetParam()) != "rocksdb") | |
410 | return; | |
411 | ||
f67539c2 | 412 | std::string cfs("cf1"); |
11fdf7f2 TL |
413 | ASSERT_EQ(0, db->init(g_conf()->bluestore_rocksdb_options)); |
414 | cout << "creating one column family and opening it" << std::endl; | |
415 | ASSERT_EQ(0, db->create_and_open(cout, cfs)); | |
416 | { | |
417 | KeyValueDB::Transaction t = db->get_transaction(); | |
418 | bufferlist bl1; | |
419 | bl1.append("hello"); | |
420 | bufferlist bl2; | |
421 | bl2.append("world"); | |
422 | cout << "write some kv pairs into default and new CFs" << std::endl; | |
423 | t->set("prefix", "key1", bl1); | |
424 | t->set("prefix", "key2", bl2); | |
425 | t->set("cf1", "key1", bl1); | |
426 | t->set("cf1", "key2", bl2); | |
427 | ASSERT_EQ(0, db->submit_transaction_sync(t)); | |
428 | } | |
429 | { | |
430 | cout << "iterating the default CF" << std::endl; | |
431 | KeyValueDB::Iterator iter = db->get_iterator("prefix"); | |
432 | iter->seek_to_first(); | |
433 | ASSERT_EQ(1, iter->valid()); | |
434 | ASSERT_EQ("key1", iter->key()); | |
435 | ASSERT_EQ("hello", _bl_to_str(iter->value())); | |
436 | ASSERT_EQ(0, iter->next()); | |
437 | ASSERT_EQ(1, iter->valid()); | |
438 | ASSERT_EQ("key2", iter->key()); | |
439 | ASSERT_EQ("world", _bl_to_str(iter->value())); | |
440 | } | |
441 | { | |
442 | cout << "iterating the new CF" << std::endl; | |
443 | KeyValueDB::Iterator iter = db->get_iterator("cf1"); | |
444 | iter->seek_to_first(); | |
445 | ASSERT_EQ(1, iter->valid()); | |
446 | ASSERT_EQ("key1", iter->key()); | |
447 | ASSERT_EQ("hello", _bl_to_str(iter->value())); | |
448 | ASSERT_EQ(0, iter->next()); | |
449 | ASSERT_EQ(1, iter->valid()); | |
450 | ASSERT_EQ("key2", iter->key()); | |
451 | ASSERT_EQ("world", _bl_to_str(iter->value())); | |
452 | } | |
453 | fini(); | |
454 | } | |
455 | ||
f67539c2 TL |
456 | TEST_P(KVTest, RocksDBShardingIteratorTest) { |
457 | if(string(GetParam()) != "rocksdb") | |
458 | return; | |
459 | ||
460 | std::string cfs("A(6)"); | |
461 | ASSERT_EQ(0, db->init(g_conf()->bluestore_rocksdb_options)); | |
462 | cout << "creating one column family and opening it" << std::endl; | |
463 | ASSERT_EQ(0, db->create_and_open(cout, cfs)); | |
464 | { | |
465 | KeyValueDB::Transaction t = db->get_transaction(); | |
466 | for (int v = 100; v <= 999; v++) { | |
467 | std::string str = to_string(v); | |
468 | bufferlist val; | |
469 | val.append(str); | |
470 | t->set("A", str, val); | |
471 | } | |
472 | ASSERT_EQ(0, db->submit_transaction_sync(t)); | |
473 | } | |
474 | { | |
475 | KeyValueDB::Iterator it = db->get_iterator("A"); | |
476 | int pos = 0; | |
477 | ASSERT_EQ(it->lower_bound(to_string(pos)), 0); | |
478 | for (pos = 100; pos <= 999; pos++) { | |
479 | ASSERT_EQ(it->valid(), true); | |
480 | ASSERT_EQ(it->key(), to_string(pos)); | |
481 | ASSERT_EQ(it->value().to_str(), to_string(pos)); | |
482 | it->next(); | |
483 | } | |
484 | ASSERT_EQ(it->valid(), false); | |
485 | pos = 999; | |
486 | ASSERT_EQ(it->lower_bound(to_string(pos)), 0); | |
487 | for (pos = 999; pos >= 100; pos--) { | |
488 | ASSERT_EQ(it->valid(), true); | |
489 | ASSERT_EQ(it->key(), to_string(pos)); | |
490 | ASSERT_EQ(it->value().to_str(), to_string(pos)); | |
491 | it->prev(); | |
492 | } | |
493 | ASSERT_EQ(it->valid(), false); | |
494 | } | |
495 | fini(); | |
496 | } | |
497 | ||
11fdf7f2 TL |
498 | TEST_P(KVTest, RocksDBCFMerge) { |
499 | if(string(GetParam()) != "rocksdb") | |
500 | return; | |
501 | ||
502 | shared_ptr<KeyValueDB::MergeOperator> p(new AppendMOP); | |
503 | int r = db->set_merge_operator("cf1",p); | |
504 | if (r < 0) | |
505 | return; // No merge operators for this database type | |
f67539c2 | 506 | std::string cfs("cf1"); |
11fdf7f2 TL |
507 | ASSERT_EQ(0, db->init(g_conf()->bluestore_rocksdb_options)); |
508 | cout << "creating one column family and opening it" << std::endl; | |
509 | ASSERT_EQ(0, db->create_and_open(cout, cfs)); | |
510 | ||
511 | { | |
512 | KeyValueDB::Transaction t = db->get_transaction(); | |
513 | bufferlist v1, v2, v3; | |
514 | v1.append(string("1")); | |
515 | v2.append(string("2")); | |
516 | v3.append(string("3")); | |
517 | t->set("P", "K1", v1); | |
518 | t->set("cf1", "A1", v2); | |
519 | t->rmkey("cf1", "A2"); | |
520 | t->merge("cf1", "A2", v3); | |
521 | db->submit_transaction_sync(t); | |
522 | } | |
523 | { | |
524 | bufferlist v1, v2, v3; | |
525 | ASSERT_EQ(0, db->get("P", "K1", &v1)); | |
526 | ASSERT_EQ(tostr(v1), "1"); | |
527 | ASSERT_EQ(0, db->get("cf1", "A1", &v2)); | |
528 | ASSERT_EQ(tostr(v2), "2"); | |
529 | ASSERT_EQ(0, db->get("cf1", "A2", &v3)); | |
530 | ASSERT_EQ(tostr(v3), "?3"); | |
531 | } | |
532 | { | |
533 | KeyValueDB::Transaction t = db->get_transaction(); | |
534 | bufferlist v1; | |
535 | v1.append(string("1")); | |
536 | t->merge("cf1", "A2", v1); | |
537 | db->submit_transaction_sync(t); | |
538 | } | |
539 | { | |
540 | bufferlist v; | |
541 | ASSERT_EQ(0, db->get("cf1", "A2", &v)); | |
542 | ASSERT_EQ(tostr(v), "?31"); | |
543 | } | |
544 | fini(); | |
545 | } | |
7c673cae | 546 | |
9f95a23c TL |
547 | TEST_P(KVTest, RocksDB_estimate_size) { |
548 | if(string(GetParam()) != "rocksdb") | |
549 | GTEST_SKIP(); | |
550 | ||
f67539c2 | 551 | std::string cfs("cf1"); |
9f95a23c TL |
552 | ASSERT_EQ(0, db->init(g_conf()->bluestore_rocksdb_options)); |
553 | cout << "creating one column family and opening it" << std::endl; | |
554 | ASSERT_EQ(0, db->create_and_open(cout)); | |
555 | ||
556 | for(int test = 0; test < 20; test++) | |
557 | { | |
558 | KeyValueDB::Transaction t = db->get_transaction(); | |
559 | bufferlist v1; | |
560 | v1.append(string(1000, '1')); | |
561 | for (int i = 0; i < 100; i++) | |
562 | t->set("A", to_string(rand()%100000), v1); | |
563 | db->submit_transaction_sync(t); | |
564 | db->compact(); | |
565 | ||
566 | int64_t size_a = db->estimate_prefix_size("A",""); | |
567 | ASSERT_GT(size_a, (test + 1) * 1000 * 100 * 0.5); | |
568 | ASSERT_LT(size_a, (test + 1) * 1000 * 100 * 1.5); | |
569 | int64_t size_a1 = db->estimate_prefix_size("A","1"); | |
570 | ASSERT_GT(size_a1, (test + 1) * 1000 * 100 * 0.1 * 0.5); | |
571 | ASSERT_LT(size_a1, (test + 1) * 1000 * 100 * 0.1 * 1.5); | |
572 | int64_t size_b = db->estimate_prefix_size("B",""); | |
573 | ASSERT_EQ(size_b, 0); | |
574 | } | |
575 | ||
576 | fini(); | |
577 | } | |
578 | ||
579 | TEST_P(KVTest, RocksDB_estimate_size_column_family) { | |
580 | if(string(GetParam()) != "rocksdb") | |
581 | GTEST_SKIP(); | |
582 | ||
f67539c2 | 583 | std::string cfs("cf1"); |
9f95a23c TL |
584 | ASSERT_EQ(0, db->init(g_conf()->bluestore_rocksdb_options)); |
585 | cout << "creating one column family and opening it" << std::endl; | |
586 | ASSERT_EQ(0, db->create_and_open(cout, cfs)); | |
587 | ||
588 | for(int test = 0; test < 20; test++) | |
589 | { | |
590 | KeyValueDB::Transaction t = db->get_transaction(); | |
591 | bufferlist v1; | |
592 | v1.append(string(1000, '1')); | |
593 | for (int i = 0; i < 100; i++) | |
594 | t->set("cf1", to_string(rand()%100000), v1); | |
595 | db->submit_transaction_sync(t); | |
596 | db->compact(); | |
7c673cae | 597 | |
9f95a23c TL |
598 | int64_t size_a = db->estimate_prefix_size("cf1",""); |
599 | ASSERT_GT(size_a, (test + 1) * 1000 * 100 * 0.5); | |
600 | ASSERT_LT(size_a, (test + 1) * 1000 * 100 * 1.5); | |
601 | int64_t size_a1 = db->estimate_prefix_size("cf1","1"); | |
602 | ASSERT_GT(size_a1, (test + 1) * 1000 * 100 * 0.1 * 0.5); | |
603 | ASSERT_LT(size_a1, (test + 1) * 1000 * 100 * 0.1 * 1.5); | |
604 | int64_t size_b = db->estimate_prefix_size("B",""); | |
605 | ASSERT_EQ(size_b, 0); | |
606 | } | |
7c673cae | 607 | |
9f95a23c TL |
608 | fini(); |
609 | } | |
7c673cae | 610 | |
f67539c2 TL |
611 | TEST_P(KVTest, RocksDB_parse_sharding_def) { |
612 | if(string(GetParam()) != "rocksdb") | |
613 | GTEST_SKIP(); | |
614 | ||
615 | bool result; | |
616 | std::vector<RocksDBStore::ColumnFamily> sharding_def; | |
617 | char const* error_position = nullptr; | |
618 | std::string error_msg; | |
619 | ||
620 | std::string_view text_def = "A(10,0-30) B(6)=option1,option2=aaaa C"; | |
621 | result = RocksDBStore::parse_sharding_def(text_def, | |
622 | sharding_def, | |
623 | &error_position, | |
624 | &error_msg); | |
625 | ||
626 | ASSERT_EQ(result, true); | |
627 | ASSERT_EQ(error_position, nullptr); | |
628 | ASSERT_EQ(error_msg, ""); | |
629 | std::cout << text_def << std::endl; | |
630 | if (error_position) std::cout << std::string(error_position - text_def.begin(), ' ') << "^" << error_msg << std::endl; | |
631 | ||
632 | ASSERT_EQ(sharding_def.size(), 3); | |
633 | ASSERT_EQ(sharding_def[0].name, "A"); | |
634 | ASSERT_EQ(sharding_def[0].shard_cnt, 10); | |
635 | ASSERT_EQ(sharding_def[0].hash_l, 0); | |
636 | ASSERT_EQ(sharding_def[0].hash_h, 30); | |
637 | ||
638 | ASSERT_EQ(sharding_def[1].name, "B"); | |
639 | ASSERT_EQ(sharding_def[1].shard_cnt, 6); | |
640 | ASSERT_EQ(sharding_def[1].options, "option1,option2=aaaa"); | |
641 | ASSERT_EQ(sharding_def[2].name, "C"); | |
642 | ASSERT_EQ(sharding_def[2].shard_cnt, 1); | |
643 | ||
644 | ||
645 | text_def = "A(10 B(6)=option C"; | |
646 | result = RocksDBStore::parse_sharding_def(text_def, | |
647 | sharding_def, | |
648 | &error_position, | |
649 | &error_msg); | |
650 | std::cout << text_def << std::endl; | |
651 | if (error_position) | |
652 | std::cout << std::string(error_position - text_def.begin(), ' ') << "^" << error_msg << std::endl; | |
653 | ASSERT_EQ(result, false); | |
654 | ASSERT_NE(error_position, nullptr); | |
655 | ASSERT_NE(error_msg, ""); | |
656 | ||
657 | text_def = "A(10,1) B(6)=option C"; | |
658 | result = RocksDBStore::parse_sharding_def(text_def, | |
659 | sharding_def, | |
660 | &error_position, | |
661 | &error_msg); | |
662 | std::cout << text_def << std::endl; | |
663 | std::cout << std::string(error_position - text_def.begin(), ' ') << "^" << error_msg << std::endl; | |
664 | ASSERT_EQ(result, false); | |
665 | ASSERT_NE(error_position, nullptr); | |
666 | ASSERT_NE(error_msg, ""); | |
667 | } | |
668 | ||
669 | ||
670 | ||
671 | class RocksDBShardingTest : public ::testing::TestWithParam<const char*> { | |
672 | public: | |
673 | boost::scoped_ptr<KeyValueDB> db; | |
674 | ||
675 | RocksDBShardingTest() : db(0) {} | |
676 | ||
677 | string _bl_to_str(bufferlist val) { | |
678 | string str(val.c_str(), val.length()); | |
679 | return str; | |
680 | } | |
681 | ||
682 | void rm_r(string path) { | |
683 | string cmd = string("rm -r ") + path; | |
684 | if (verbose) | |
685 | cout << "==> " << cmd << std::endl; | |
686 | int r = ::system(cmd.c_str()); | |
687 | if (r) { | |
688 | cerr << "failed with exit code " << r | |
689 | << ", continuing anyway" << std::endl; | |
690 | } | |
691 | } | |
692 | ||
693 | void SetUp() override { | |
694 | verbose = getenv("VERBOSE") && strcmp(getenv("VERBOSE"), "1") == 0; | |
695 | ||
696 | int r = ::mkdir("kv_test_temp_dir", 0777); | |
697 | if (r < 0 && errno != EEXIST) { | |
698 | r = -errno; | |
699 | cerr << __func__ << ": unable to create kv_test_temp_dir: " | |
700 | << cpp_strerror(r) << std::endl; | |
701 | return; | |
702 | } | |
703 | db.reset(KeyValueDB::create(g_ceph_context, "rocksdb", | |
704 | "kv_test_temp_dir")); | |
705 | ASSERT_EQ(0, db->init(g_conf()->bluestore_rocksdb_options)); | |
706 | if (verbose) | |
707 | cout << "Creating database with sharding: " << GetParam() << std::endl; | |
708 | ASSERT_EQ(0, db->create_and_open(cout, GetParam())); | |
709 | } | |
710 | void TearDown() override { | |
711 | db.reset(nullptr); | |
712 | rm_r("kv_test_temp_dir"); | |
713 | } | |
714 | ||
715 | /* | |
716 | A - main 0/1/20 | |
717 | B - shard 1/3 x 0/1/20 | |
718 | C - main 0/1/20 | |
719 | D - shard 1/3 x 0/1/20 | |
720 | E - main 0/1/20 | |
721 | */ | |
722 | bool verbose; | |
723 | std::vector<std::string> sharding_defs = { | |
724 | "Betelgeuse D", | |
725 | "Betelgeuse(3) D", | |
726 | "Betelgeuse D(3)", | |
727 | "Betelgeuse(3) D(3)"}; | |
728 | std::vector<std::string> prefixes = {"Ad", "Betelgeuse", "C", "D", "Evade"}; | |
729 | std::vector<std::string> randoms = {"0", "1", "2", "3", "4", "5", | |
730 | "found", "brain", "fully", "pen", "worth", "race", | |
731 | "stand", "nodded", "whenever", "surrounded", "industrial", "skin", | |
732 | "this", "direction", "family", "beginning", "whenever", "held", | |
733 | "metal", "year", "like", "valuable", "softly", "whistle", | |
734 | "perfectly", "broken", "idea", "also", "coffee", "branch", | |
735 | "tongue", "immediately", "bent", "partly", "burn", "include", | |
736 | "certain", "burst", "final", "smoke", "positive", "perfectly" | |
737 | }; | |
738 | int R = randoms.size(); | |
739 | ||
740 | typedef int test_id[6]; | |
741 | void zero(test_id& x) { | |
742 | k = 0; | |
743 | v = 0; | |
744 | for (auto& i:x) | |
745 | i = 0; | |
746 | } | |
747 | bool end(const test_id& x) { | |
748 | return x[5] != 0; | |
749 | } | |
750 | void next(test_id& x) { | |
751 | x[0]++; | |
752 | for (int i = 0; i < 5; i++) { | |
753 | if (x[i] == 3) { | |
754 | x[i] = 0; | |
755 | ++x[i + 1]; | |
756 | } | |
757 | } | |
758 | } | |
759 | ||
760 | std::map<std::string, std::string> data; | |
761 | int k = 0; | |
762 | int v = 0; | |
763 | ||
764 | void generate_data(const test_id& x) { | |
765 | data.clear(); | |
766 | for (int i = 0; i < 5; i++) { | |
767 | if (verbose) | |
768 | std::cout << x[i] << "-"; | |
769 | switch (x[i]) { | |
770 | case 0: | |
771 | break; | |
772 | case 1: | |
773 | data[RocksDBStore::combine_strings(prefixes[i], randoms[k++ % R])] = randoms[v++ % R]; | |
774 | break; | |
775 | case 2: | |
776 | std::string base = randoms[k++ % R]; | |
777 | for (int j = 0; j < 10; j++) { | |
778 | data[RocksDBStore::combine_strings(prefixes[i], base + "." + randoms[k++ % R])] = randoms[v++ % R]; | |
779 | } | |
780 | break; | |
781 | } | |
782 | } | |
783 | } | |
784 | ||
785 | void data_to_db() { | |
786 | KeyValueDB::Transaction t = db->get_transaction(); | |
787 | for (auto &d : data) { | |
788 | bufferlist v1; | |
789 | v1.append(d.second); | |
790 | string prefix; | |
791 | string key; | |
792 | RocksDBStore::split_key(d.first, &prefix, &key); | |
793 | t->set(prefix, key, v1); | |
794 | if (verbose) | |
795 | std::cout << "SET " << prefix << " " << key << std::endl; | |
796 | } | |
797 | ASSERT_EQ(db->submit_transaction_sync(t), 0); | |
798 | } | |
799 | ||
800 | void clear_db() { | |
801 | KeyValueDB::Transaction t = db->get_transaction(); | |
802 | for (auto &d : data) { | |
803 | string prefix; | |
804 | string key; | |
805 | RocksDBStore::split_key(d.first, &prefix, &key); | |
806 | t->rmkey(prefix, key); | |
807 | } | |
808 | ASSERT_EQ(db->submit_transaction_sync(t), 0); | |
809 | //paranoid, check if db empty | |
810 | KeyValueDB::WholeSpaceIterator it = db->get_wholespace_iterator(); | |
811 | ASSERT_EQ(it->seek_to_first(), 0); | |
812 | ASSERT_EQ(it->valid(), false); | |
813 | } | |
814 | }; | |
815 | ||
816 | TEST_P(RocksDBShardingTest, wholespace_next) { | |
817 | test_id X; | |
818 | zero(X); | |
819 | do { | |
820 | generate_data(X); | |
821 | data_to_db(); | |
822 | ||
823 | KeyValueDB::WholeSpaceIterator it = db->get_wholespace_iterator(); | |
824 | //move forward | |
825 | auto dit = data.begin(); | |
826 | int r = it->seek_to_first(); | |
827 | ASSERT_EQ(r, 0); | |
828 | ASSERT_EQ(it->valid(), (dit != data.end())); | |
829 | ||
830 | while (dit != data.end()) { | |
831 | ASSERT_EQ(it->valid(), true); | |
832 | string prefix; | |
833 | string key; | |
834 | RocksDBStore::split_key(dit->first, &prefix, &key); | |
835 | auto raw_key = it->raw_key(); | |
836 | ASSERT_EQ(raw_key.first, prefix); | |
837 | ASSERT_EQ(raw_key.second, key); | |
838 | ASSERT_EQ(it->value().to_str(), dit->second); | |
839 | if (verbose) | |
840 | std::cout << "next " << prefix << " " << key << std::endl; | |
841 | ASSERT_EQ(it->next(), 0); | |
842 | ++dit; | |
843 | } | |
844 | ASSERT_EQ(it->valid(), false); | |
845 | ||
846 | clear_db(); | |
847 | next(X); | |
848 | } while (!end(X)); | |
849 | } | |
850 | ||
851 | TEST_P(RocksDBShardingTest, wholespace_prev) { | |
852 | test_id X; | |
853 | zero(X); | |
854 | do { | |
855 | generate_data(X); | |
856 | data_to_db(); | |
857 | ||
858 | KeyValueDB::WholeSpaceIterator it = db->get_wholespace_iterator(); | |
859 | auto dit = data.rbegin(); | |
860 | int r = it->seek_to_last(); | |
861 | ASSERT_EQ(r, 0); | |
862 | ASSERT_EQ(it->valid(), (dit != data.rend())); | |
863 | ||
864 | while (dit != data.rend()) { | |
865 | ASSERT_EQ(it->valid(), true); | |
866 | string prefix; | |
867 | string key; | |
868 | RocksDBStore::split_key(dit->first, &prefix, &key); | |
869 | auto raw_key = it->raw_key(); | |
870 | ASSERT_EQ(raw_key.first, prefix); | |
871 | ASSERT_EQ(raw_key.second, key); | |
872 | ASSERT_EQ(it->value().to_str(), dit->second); | |
873 | if (verbose) | |
874 | std::cout << "prev " << prefix << " " << key << std::endl; | |
875 | ASSERT_EQ(it->prev(), 0); | |
876 | ++dit; | |
877 | } | |
878 | ASSERT_EQ(it->valid(), false); | |
879 | ||
880 | clear_db(); | |
881 | next(X); | |
882 | } while (!end(X)); | |
883 | } | |
884 | ||
885 | TEST_P(RocksDBShardingTest, wholespace_lower_bound) { | |
886 | test_id X; | |
887 | zero(X); | |
888 | do { | |
889 | generate_data(X); | |
890 | data_to_db(); | |
891 | ||
892 | KeyValueDB::WholeSpaceIterator it = db->get_wholespace_iterator(); | |
893 | auto dit = data.begin(); | |
894 | int r = it->seek_to_first(); | |
895 | ASSERT_EQ(r, 0); | |
896 | ASSERT_EQ(it->valid(), (dit != data.end())); | |
897 | ||
898 | while (dit != data.end()) { | |
899 | ASSERT_EQ(it->valid(), true); | |
900 | string prefix; | |
901 | string key; | |
902 | RocksDBStore::split_key(dit->first, &prefix, &key); | |
903 | KeyValueDB::WholeSpaceIterator it1 = db->get_wholespace_iterator(); | |
904 | ASSERT_EQ(it1->lower_bound(prefix, key), 0); | |
905 | ASSERT_EQ(it1->valid(), true); | |
906 | auto raw_key = it1->raw_key(); | |
907 | ASSERT_EQ(raw_key.first, prefix); | |
908 | ASSERT_EQ(raw_key.second, key); | |
909 | if (verbose) | |
910 | std::cout << "lower_bound " << prefix << " " << key << std::endl; | |
911 | ASSERT_EQ(it->next(), 0); | |
912 | ++dit; | |
913 | } | |
914 | ASSERT_EQ(it->valid(), false); | |
915 | ||
916 | clear_db(); | |
917 | next(X); | |
918 | } while (!end(X)); | |
919 | } | |
920 | ||
921 | TEST_P(RocksDBShardingTest, wholespace_upper_bound) { | |
922 | test_id X; | |
923 | zero(X); | |
924 | do { | |
925 | generate_data(X); | |
926 | data_to_db(); | |
927 | ||
928 | KeyValueDB::WholeSpaceIterator it = db->get_wholespace_iterator(); | |
929 | auto dit = data.begin(); | |
930 | int r = it->seek_to_first(); | |
931 | ASSERT_EQ(r, 0); | |
932 | ASSERT_EQ(it->valid(), (dit != data.end())); | |
933 | ||
934 | while (dit != data.end()) { | |
935 | ASSERT_EQ(it->valid(), true); | |
936 | string prefix; | |
937 | string key; | |
938 | string key_minus_1; | |
939 | RocksDBStore::split_key(dit->first, &prefix, &key); | |
940 | //decrement key minimally | |
941 | key_minus_1 = key.substr(0, key.length() - 1) + std::string(1, key[key.length() - 1] - 1); | |
942 | KeyValueDB::WholeSpaceIterator it1 = db->get_wholespace_iterator(); | |
943 | ASSERT_EQ(it1->upper_bound(prefix, key_minus_1), 0); | |
944 | ASSERT_EQ(it1->valid(), true); | |
945 | auto raw_key = it1->raw_key(); | |
946 | ASSERT_EQ(raw_key.first, prefix); | |
947 | ASSERT_EQ(raw_key.second, key); | |
948 | if (verbose) | |
949 | std::cout << "upper_bound " << prefix << " " << key_minus_1 << std::endl; | |
950 | ASSERT_EQ(it->next(), 0); | |
951 | ++dit; | |
952 | } | |
953 | ASSERT_EQ(it->valid(), false); | |
954 | ||
955 | clear_db(); | |
956 | next(X); | |
957 | } while (!end(X)); | |
958 | } | |
959 | ||
960 | TEST_P(RocksDBShardingTest, wholespace_lookup_limits) { | |
961 | test_id X; | |
962 | zero(X); | |
963 | do { | |
964 | generate_data(X); | |
965 | data_to_db(); | |
966 | ||
967 | //lookup before first | |
968 | if (data.size() > 0) { | |
969 | auto dit = data.begin(); | |
970 | string prefix; | |
971 | string key; | |
972 | RocksDBStore::split_key(dit->first, &prefix, &key); | |
973 | KeyValueDB::WholeSpaceIterator it1 = db->get_wholespace_iterator(); | |
974 | ASSERT_EQ(it1->lower_bound(" ", " "), 0); | |
975 | ASSERT_EQ(it1->valid(), true); | |
976 | auto raw_key = it1->raw_key(); | |
977 | ASSERT_EQ(raw_key.first, prefix); | |
978 | ASSERT_EQ(raw_key.second, key); | |
979 | } | |
980 | //lookup after last | |
981 | KeyValueDB::WholeSpaceIterator it1 = db->get_wholespace_iterator(); | |
982 | ASSERT_EQ(it1->lower_bound("~", "~"), 0); | |
983 | ASSERT_EQ(it1->valid(), false); | |
984 | ||
985 | clear_db(); | |
986 | next(X); | |
987 | } while (!end(X)); | |
988 | } | |
989 | ||
990 | ||
991 | ||
992 | class RocksDBResharding : public ::testing::Test { | |
993 | public: | |
994 | boost::scoped_ptr<RocksDBStore> db; | |
995 | ||
996 | RocksDBResharding() : db(0) {} | |
997 | ||
998 | string _bl_to_str(bufferlist val) { | |
999 | string str(val.c_str(), val.length()); | |
1000 | return str; | |
1001 | } | |
1002 | ||
1003 | void rm_r(string path) { | |
1004 | string cmd = string("rm -r ") + path; | |
1005 | if (verbose) | |
1006 | cout << "==> " << cmd << std::endl; | |
1007 | int r = ::system(cmd.c_str()); | |
1008 | if (r) { | |
1009 | cerr << "failed with exit code " << r | |
1010 | << ", continuing anyway" << std::endl; | |
1011 | } | |
1012 | } | |
1013 | ||
1014 | void SetUp() override { | |
1015 | verbose = getenv("VERBOSE") && strcmp(getenv("VERBOSE"), "1") == 0; | |
1016 | ||
1017 | int r = ::mkdir("kv_test_temp_dir", 0777); | |
1018 | if (r < 0 && errno != EEXIST) { | |
1019 | r = -errno; | |
1020 | cerr << __func__ << ": unable to create kv_test_temp_dir: " | |
1021 | << cpp_strerror(r) << std::endl; | |
1022 | return; | |
1023 | } | |
1024 | ||
1025 | KeyValueDB* db_kv = KeyValueDB::create(g_ceph_context, "rocksdb", | |
1026 | "kv_test_temp_dir"); | |
1027 | RocksDBStore* db_rocks = dynamic_cast<RocksDBStore*>(db_kv); | |
1028 | ceph_assert(db_rocks); | |
1029 | db.reset(db_rocks); | |
1030 | ASSERT_EQ(0, db->init(g_conf()->bluestore_rocksdb_options)); | |
1031 | } | |
1032 | void TearDown() override { | |
1033 | db.reset(nullptr); | |
1034 | rm_r("kv_test_temp_dir"); | |
1035 | } | |
1036 | ||
1037 | bool verbose; | |
1038 | std::vector<std::string> prefixes = {"Ad", "Betelgeuse", "C", "D", "Evade"}; | |
1039 | std::vector<std::string> randoms = {"0", "1", "2", "3", "4", "5", | |
1040 | "found", "brain", "fully", "pen", "worth", "race", | |
1041 | "stand", "nodded", "whenever", "surrounded", "industrial", "skin", | |
1042 | "this", "direction", "family", "beginning", "whenever", "held", | |
1043 | "metal", "year", "like", "valuable", "softly", "whistle", | |
1044 | "perfectly", "broken", "idea", "also", "coffee", "branch", | |
1045 | "tongue", "immediately", "bent", "partly", "burn", "include", | |
1046 | "certain", "burst", "final", "smoke", "positive", "perfectly" | |
1047 | }; | |
1048 | int R = randoms.size(); | |
1049 | int k = 0; | |
1050 | std::map<std::string, std::string> data; | |
1051 | ||
1052 | void generate_data() { | |
1053 | data.clear(); | |
1054 | for (size_t p = 0; p < prefixes.size(); p++) { | |
1055 | size_t elem_count = 1 << (( p * 3 ) + 3); | |
1056 | for (size_t i = 0; i < elem_count; i++) { | |
1057 | std::string key; | |
1058 | for (int x = 0; x < 5; x++) { | |
1059 | key = key + randoms[rand() % R]; | |
1060 | } | |
1061 | std::string value; | |
1062 | for (int x = 0; x < 3; x++) { | |
1063 | value = value + randoms[rand() % R]; | |
1064 | } | |
1065 | data[RocksDBStore::combine_strings(prefixes[p], key)] = value; | |
1066 | } | |
1067 | } | |
1068 | } | |
1069 | ||
1070 | void data_to_db() { | |
1071 | KeyValueDB::Transaction t = db->get_transaction(); | |
1072 | size_t i = 0; | |
1073 | for (auto& d: data) { | |
1074 | bufferlist v1; | |
1075 | v1.append(d.second); | |
1076 | string prefix; | |
1077 | string key; | |
1078 | RocksDBStore::split_key(d.first, &prefix, &key); | |
1079 | t->set(prefix, key, v1); | |
1080 | if (verbose) | |
1081 | std::cout << "SET " << prefix << " " << key << std::endl; | |
1082 | i++; | |
1083 | if ((i % 1000) == 0) { | |
1084 | ASSERT_EQ(db->submit_transaction_sync(t), 0); | |
1085 | t.reset(); | |
1086 | if (verbose) | |
1087 | std::cout << "writing key to DB" << std::endl; | |
1088 | t = db->get_transaction(); | |
1089 | } | |
1090 | } | |
1091 | if (verbose) | |
1092 | std::cout << "writing keys to DB" << std::endl; | |
1093 | ASSERT_EQ(db->submit_transaction_sync(t), 0); | |
1094 | } | |
1095 | ||
1096 | void clear_db() { | |
1097 | KeyValueDB::Transaction t = db->get_transaction(); | |
1098 | for (auto &d : data) { | |
1099 | string prefix; | |
1100 | string key; | |
1101 | RocksDBStore::split_key(d.first, &prefix, &key); | |
1102 | t->rmkey(prefix, key); | |
1103 | } | |
1104 | ASSERT_EQ(db->submit_transaction_sync(t), 0); | |
1105 | //paranoid, check if db empty | |
1106 | KeyValueDB::WholeSpaceIterator it = db->get_wholespace_iterator(); | |
1107 | ASSERT_EQ(it->seek_to_first(), 0); | |
1108 | ASSERT_EQ(it->valid(), false); | |
1109 | } | |
1110 | ||
1111 | void check_db() { | |
1112 | KeyValueDB::WholeSpaceIterator it = db->get_wholespace_iterator(); | |
1113 | //move forward | |
1114 | auto dit = data.begin(); | |
1115 | int r = it->seek_to_first(); | |
1116 | ASSERT_EQ(r, 0); | |
1117 | ASSERT_EQ(it->valid(), (dit != data.end())); | |
1118 | ||
1119 | while (dit != data.end()) { | |
1120 | ASSERT_EQ(it->valid(), true); | |
1121 | string prefix; | |
1122 | string key; | |
1123 | RocksDBStore::split_key(dit->first, &prefix, &key); | |
1124 | auto raw_key = it->raw_key(); | |
1125 | ASSERT_EQ(raw_key.first, prefix); | |
1126 | ASSERT_EQ(raw_key.second, key); | |
1127 | ASSERT_EQ(it->value().to_str(), dit->second); | |
1128 | if (verbose) | |
1129 | std::cout << "next " << prefix << " " << key << std::endl; | |
1130 | ASSERT_EQ(it->next(), 0); | |
1131 | ++dit; | |
1132 | } | |
1133 | ASSERT_EQ(it->valid(), false); | |
1134 | } | |
1135 | }; | |
1136 | ||
1137 | TEST_F(RocksDBResharding, basic) { | |
1138 | ASSERT_EQ(0, db->create_and_open(cout, "")); | |
1139 | generate_data(); | |
1140 | data_to_db(); | |
1141 | check_db(); | |
1142 | db->close(); | |
1143 | ASSERT_EQ(db->reshard("Evade(4)"), 0); | |
1144 | ASSERT_EQ(db->open(cout), 0); | |
1145 | check_db(); | |
1146 | db->close(); | |
1147 | } | |
1148 | ||
1149 | TEST_F(RocksDBResharding, all_to_shards) { | |
1150 | ASSERT_EQ(0, db->create_and_open(cout, "")); | |
1151 | generate_data(); | |
1152 | data_to_db(); | |
1153 | check_db(); | |
1154 | db->close(); | |
1155 | ASSERT_EQ(db->reshard("Ad(1) Betelgeuse(1) C(1) D(1) Evade(1)"), 0); | |
1156 | ASSERT_EQ(db->open(cout), 0); | |
1157 | check_db(); | |
1158 | db->close(); | |
1159 | } | |
1160 | ||
1161 | TEST_F(RocksDBResharding, all_to_shards_and_back_again) { | |
1162 | ASSERT_EQ(0, db->create_and_open(cout, "")); | |
1163 | generate_data(); | |
1164 | data_to_db(); | |
1165 | check_db(); | |
1166 | db->close(); | |
1167 | ASSERT_EQ(db->reshard("Ad(1) Betelgeuse(1) C(1) D(1) Evade(1)"), 0); | |
1168 | ASSERT_EQ(db->open(cout), 0); | |
1169 | check_db(); | |
1170 | db->close(); | |
1171 | ASSERT_EQ(db->reshard(""), 0); | |
1172 | ASSERT_EQ(db->open(cout), 0); | |
1173 | check_db(); | |
1174 | db->close(); | |
1175 | } | |
1176 | ||
1177 | TEST_F(RocksDBResharding, resume_interrupted_at_batch) { | |
1178 | ASSERT_EQ(0, db->create_and_open(cout, "")); | |
1179 | generate_data(); | |
1180 | data_to_db(); | |
1181 | check_db(); | |
1182 | db->close(); | |
1183 | RocksDBStore::resharding_ctrl ctrl; | |
1184 | ctrl.unittest_fail_after_first_batch = true; | |
1185 | ASSERT_EQ(db->reshard("Evade(4)", &ctrl), -1000); | |
1186 | ASSERT_NE(db->open(cout), 0); | |
1187 | ASSERT_EQ(db->reshard("Evade(4)"), 0); | |
1188 | ASSERT_EQ(db->open(cout), 0); | |
1189 | check_db(); | |
1190 | db->close(); | |
1191 | } | |
1192 | ||
1193 | TEST_F(RocksDBResharding, resume_interrupted_at_column) { | |
1194 | ASSERT_EQ(0, db->create_and_open(cout, "")); | |
1195 | generate_data(); | |
1196 | data_to_db(); | |
1197 | check_db(); | |
1198 | db->close(); | |
1199 | RocksDBStore::resharding_ctrl ctrl; | |
1200 | ctrl.unittest_fail_after_processing_column = true; | |
1201 | ASSERT_EQ(db->reshard("Evade(4)", &ctrl), -1001); | |
1202 | ASSERT_NE(db->open(cout), 0); | |
1203 | ASSERT_EQ(db->reshard("Evade(4)"), 0); | |
1204 | ASSERT_EQ(db->open(cout), 0); | |
1205 | check_db(); | |
1206 | db->close(); | |
1207 | } | |
1208 | ||
1209 | TEST_F(RocksDBResharding, resume_interrupted_before_commit) { | |
1210 | ASSERT_EQ(0, db->create_and_open(cout, "")); | |
1211 | generate_data(); | |
1212 | data_to_db(); | |
1213 | check_db(); | |
1214 | db->close(); | |
1215 | RocksDBStore::resharding_ctrl ctrl; | |
1216 | ctrl.unittest_fail_after_successful_processing = true; | |
1217 | ASSERT_EQ(db->reshard("Evade(4)", &ctrl), -1002); | |
1218 | ASSERT_NE(db->open(cout), 0); | |
1219 | ASSERT_EQ(db->reshard("Evade(4)"), 0); | |
1220 | ASSERT_EQ(db->open(cout), 0); | |
1221 | check_db(); | |
1222 | db->close(); | |
1223 | } | |
1224 | ||
1225 | TEST_F(RocksDBResharding, prevent_incomplete_hash_change) { | |
1226 | ASSERT_EQ(0, db->create_and_open(cout, "Evade(4,0-3)")); | |
1227 | generate_data(); | |
1228 | data_to_db(); | |
1229 | check_db(); | |
1230 | db->close(); | |
1231 | RocksDBStore::resharding_ctrl ctrl; | |
1232 | ctrl.unittest_fail_after_successful_processing = true; | |
1233 | ASSERT_EQ(db->reshard("Evade(4,0-8)", &ctrl), -1002); | |
1234 | ASSERT_NE(db->open(cout), 0); | |
1235 | ASSERT_EQ(db->reshard("Evade(4,0-8)"), 0); | |
1236 | ASSERT_EQ(db->open(cout), 0); | |
1237 | check_db(); | |
1238 | db->close(); | |
1239 | } | |
1240 | ||
1241 | TEST_F(RocksDBResharding, change_reshard) { | |
1242 | ASSERT_EQ(0, db->create_and_open(cout, "Ad(4)")); | |
1243 | generate_data(); | |
1244 | data_to_db(); | |
1245 | check_db(); | |
1246 | db->close(); | |
1247 | { | |
1248 | RocksDBStore::resharding_ctrl ctrl; | |
1249 | ctrl.unittest_fail_after_first_batch = true; | |
1250 | ASSERT_EQ(db->reshard("C(5) D(3)", &ctrl), -1000); | |
1251 | } | |
1252 | { | |
1253 | RocksDBStore::resharding_ctrl ctrl; | |
1254 | ASSERT_NE(db->open(cout), 0); | |
1255 | ctrl.unittest_fail_after_first_batch = false; | |
1256 | ctrl.unittest_fail_after_processing_column = true; | |
1257 | ASSERT_EQ(db->reshard("C(5) Evade(2)", &ctrl), -1001); | |
1258 | } | |
1259 | { | |
1260 | RocksDBStore::resharding_ctrl ctrl; | |
1261 | ASSERT_NE(db->open(cout), 0); | |
1262 | ctrl.unittest_fail_after_processing_column = false; | |
1263 | ctrl.unittest_fail_after_successful_processing = true; | |
1264 | ASSERT_EQ(db->reshard("Evade(2) D(3)", &ctrl), -1002); | |
1265 | } | |
1266 | { | |
1267 | ASSERT_NE(db->open(cout), 0); | |
1268 | ASSERT_EQ(db->reshard("Ad(1) Evade(5)"), 0); | |
1269 | } | |
1270 | { | |
1271 | ASSERT_EQ(db->open(cout), 0); | |
1272 | check_db(); | |
1273 | db->close(); | |
1274 | } | |
1275 | } | |
1276 | ||
1277 | ||
9f95a23c TL |
1278 | INSTANTIATE_TEST_SUITE_P( |
1279 | KeyValueDB, | |
1280 | KVTest, | |
20effc67 | 1281 | ::testing::Values("rocksdb", "memdb")); |
7c673cae | 1282 | |
f67539c2 TL |
1283 | INSTANTIATE_TEST_SUITE_P( |
1284 | KeyValueDB, | |
1285 | RocksDBShardingTest, | |
1286 | ::testing::Values("Betelgeuse D", | |
1287 | "Betelgeuse(3) D", | |
1288 | "Betelgeuse D(3)", | |
1289 | "Betelgeuse(3) D(3)")); | |
1290 | ||
7c673cae | 1291 | int main(int argc, char **argv) { |
20effc67 | 1292 | auto args = argv_to_vec(argc, argv); |
7c673cae | 1293 | auto cct = global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, |
11fdf7f2 TL |
1294 | CODE_ENVIRONMENT_UTILITY, |
1295 | CINIT_FLAG_NO_DEFAULT_CONFIG_FILE); | |
7c673cae | 1296 | common_init_finish(g_ceph_context); |
11fdf7f2 | 1297 | g_ceph_context->_conf.set_val( |
7c673cae FG |
1298 | "enable_experimental_unrecoverable_data_corrupting_features", |
1299 | "rocksdb, memdb"); | |
11fdf7f2 | 1300 | g_ceph_context->_conf.apply_changes(nullptr); |
7c673cae FG |
1301 | |
1302 | ::testing::InitGoogleTest(&argc, argv); | |
1303 | return RUN_ALL_TESTS(); | |
1304 | } |