]>
Commit | Line | Data |
---|---|---|
1 | // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- | |
2 | // vim: ts=8 sw=2 smarttab | |
3 | /* | |
4 | * Ceph - scalable distributed file system | |
5 | * | |
6 | * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net> | |
7 | * | |
8 | * This is free software; you can redistribute it and/or | |
9 | * modify it under the terms of the GNU Lesser General Public | |
10 | * License version 2.1, as published by the Free Software | |
11 | * Foundation. See file COPYING. | |
12 | * | |
13 | */ | |
14 | ||
15 | #include <stdio.h> | |
16 | #include <string.h> | |
17 | #include <iostream> | |
18 | #include <time.h> | |
19 | #include <sys/mount.h> | |
20 | #include "kv/KeyValueDB.h" | |
21 | #include "include/Context.h" | |
22 | #include "common/ceph_argparse.h" | |
23 | #include "global/global_init.h" | |
24 | #include "common/Mutex.h" | |
25 | #include "common/Cond.h" | |
26 | #include "common/errno.h" | |
27 | #include "include/stringify.h" | |
28 | #include <gtest/gtest.h> | |
29 | ||
30 | #if GTEST_HAS_PARAM_TEST | |
31 | ||
32 | class KVTest : public ::testing::TestWithParam<const char*> { | |
33 | public: | |
34 | boost::scoped_ptr<KeyValueDB> db; | |
35 | ||
36 | KVTest() : db(0) {} | |
37 | ||
38 | string _bl_to_str(bufferlist val) { | |
39 | string str(val.c_str(), val.length()); | |
40 | return str; | |
41 | } | |
42 | ||
43 | void rm_r(string path) { | |
44 | string cmd = string("rm -r ") + path; | |
45 | cout << "==> " << cmd << std::endl; | |
46 | int r = ::system(cmd.c_str()); | |
47 | if (r) { | |
48 | cerr << "failed with exit code " << r | |
49 | << ", continuing anyway" << std::endl; | |
50 | } | |
51 | } | |
52 | ||
53 | void init() { | |
54 | cout << "Creating " << string(GetParam()) << "\n"; | |
55 | db.reset(KeyValueDB::create(g_ceph_context, string(GetParam()), | |
56 | "kv_test_temp_dir")); | |
57 | } | |
58 | void fini() { | |
59 | db.reset(NULL); | |
60 | } | |
61 | ||
62 | void SetUp() override { | |
63 | int r = ::mkdir("kv_test_temp_dir", 0777); | |
64 | if (r < 0 && errno != EEXIST) { | |
65 | r = -errno; | |
66 | cerr << __func__ << ": unable to create kv_test_temp_dir: " | |
67 | << cpp_strerror(r) << std::endl; | |
68 | return; | |
69 | } | |
70 | init(); | |
71 | } | |
72 | void TearDown() override { | |
73 | fini(); | |
74 | rm_r("kv_test_temp_dir"); | |
75 | } | |
76 | }; | |
77 | ||
78 | TEST_P(KVTest, OpenClose) { | |
79 | ASSERT_EQ(0, db->create_and_open(cout)); | |
80 | fini(); | |
81 | } | |
82 | ||
83 | TEST_P(KVTest, OpenCloseReopenClose) { | |
84 | ASSERT_EQ(0, db->create_and_open(cout)); | |
85 | fini(); | |
86 | init(); | |
87 | ASSERT_EQ(0, db->open(cout)); | |
88 | fini(); | |
89 | } | |
90 | ||
91 | /* | |
92 | * Basic write and read test case in same database session. | |
93 | */ | |
94 | TEST_P(KVTest, OpenWriteRead) { | |
95 | ASSERT_EQ(0, db->create_and_open(cout)); | |
96 | { | |
97 | KeyValueDB::Transaction t = db->get_transaction(); | |
98 | bufferlist value; | |
99 | value.append("value"); | |
100 | t->set("prefix", "key", value); | |
101 | value.clear(); | |
102 | value.append("value2"); | |
103 | t->set("prefix", "key2", value); | |
104 | value.clear(); | |
105 | value.append("value3"); | |
106 | t->set("prefix", "key3", value); | |
107 | db->submit_transaction_sync(t); | |
108 | ||
109 | bufferlist v1, v2; | |
110 | ASSERT_EQ(0, db->get("prefix", "key", &v1)); | |
111 | ASSERT_EQ(v1.length(), 5u); | |
112 | (v1.c_str())[v1.length()] = 0x0; | |
113 | ASSERT_EQ(std::string(v1.c_str()), std::string("value")); | |
114 | ASSERT_EQ(0, db->get("prefix", "key2", &v2)); | |
115 | ASSERT_EQ(v2.length(), 6u); | |
116 | (v2.c_str())[v2.length()] = 0x0; | |
117 | ASSERT_EQ(std::string(v2.c_str()), std::string("value2")); | |
118 | } | |
119 | fini(); | |
120 | } | |
121 | ||
122 | TEST_P(KVTest, PutReopen) { | |
123 | ASSERT_EQ(0, db->create_and_open(cout)); | |
124 | { | |
125 | KeyValueDB::Transaction t = db->get_transaction(); | |
126 | bufferlist value; | |
127 | value.append("value"); | |
128 | t->set("prefix", "key", value); | |
129 | t->set("prefix", "key2", value); | |
130 | t->set("prefix", "key3", value); | |
131 | db->submit_transaction_sync(t); | |
132 | } | |
133 | fini(); | |
134 | ||
135 | init(); | |
136 | ASSERT_EQ(0, db->open(cout)); | |
137 | { | |
138 | bufferlist v1, v2; | |
139 | ASSERT_EQ(0, db->get("prefix", "key", &v1)); | |
140 | ASSERT_EQ(v1.length(), 5u); | |
141 | ASSERT_EQ(0, db->get("prefix", "key2", &v2)); | |
142 | ASSERT_EQ(v2.length(), 5u); | |
143 | } | |
144 | { | |
145 | KeyValueDB::Transaction t = db->get_transaction(); | |
146 | t->rmkey("prefix", "key"); | |
147 | t->rmkey("prefix", "key3"); | |
148 | db->submit_transaction_sync(t); | |
149 | } | |
150 | fini(); | |
151 | ||
152 | init(); | |
153 | ASSERT_EQ(0, db->open(cout)); | |
154 | { | |
155 | bufferlist v1, v2, v3; | |
156 | ASSERT_EQ(-ENOENT, db->get("prefix", "key", &v1)); | |
157 | ASSERT_EQ(0, db->get("prefix", "key2", &v2)); | |
158 | ASSERT_EQ(v2.length(), 5u); | |
159 | ASSERT_EQ(-ENOENT, db->get("prefix", "key3", &v3)); | |
160 | } | |
161 | fini(); | |
162 | } | |
163 | ||
164 | TEST_P(KVTest, BenchCommit) { | |
165 | int n = 1024; | |
166 | ASSERT_EQ(0, db->create_and_open(cout)); | |
167 | utime_t start = ceph_clock_now(); | |
168 | { | |
169 | cout << "priming" << std::endl; | |
170 | // prime | |
171 | bufferlist big; | |
172 | bufferptr bp(1048576); | |
173 | bp.zero(); | |
174 | big.append(bp); | |
175 | for (int i=0; i<30; ++i) { | |
176 | KeyValueDB::Transaction t = db->get_transaction(); | |
177 | t->set("prefix", "big" + stringify(i), big); | |
178 | db->submit_transaction_sync(t); | |
179 | } | |
180 | } | |
181 | cout << "now doing small writes" << std::endl; | |
182 | bufferlist data; | |
183 | bufferptr bp(1024); | |
184 | bp.zero(); | |
185 | data.append(bp); | |
186 | for (int i=0; i<n; ++i) { | |
187 | KeyValueDB::Transaction t = db->get_transaction(); | |
188 | t->set("prefix", "key" + stringify(i), data); | |
189 | db->submit_transaction_sync(t); | |
190 | } | |
191 | utime_t end = ceph_clock_now(); | |
192 | utime_t dur = end - start; | |
193 | cout << n << " commits in " << dur << ", avg latency " << (dur / (double)n) | |
194 | << std::endl; | |
195 | fini(); | |
196 | } | |
197 | ||
198 | struct AppendMOP : public KeyValueDB::MergeOperator { | |
199 | void merge_nonexistent( | |
200 | const char *rdata, size_t rlen, std::string *new_value) override { | |
201 | *new_value = "?" + std::string(rdata, rlen); | |
202 | } | |
203 | void merge( | |
204 | const char *ldata, size_t llen, | |
205 | const char *rdata, size_t rlen, | |
206 | std::string *new_value) override { | |
207 | *new_value = std::string(ldata, llen) + std::string(rdata, rlen); | |
208 | } | |
209 | // We use each operator name and each prefix to construct the | |
210 | // overall RocksDB operator name for consistency check at open time. | |
211 | const char *name() const override { | |
212 | return "Append"; | |
213 | } | |
214 | }; | |
215 | ||
216 | string tostr(bufferlist& b) { | |
217 | return string(b.c_str(),b.length()); | |
218 | } | |
219 | ||
220 | TEST_P(KVTest, Merge) { | |
221 | shared_ptr<KeyValueDB::MergeOperator> p(new AppendMOP); | |
222 | int r = db->set_merge_operator("A",p); | |
223 | if (r < 0) | |
224 | return; // No merge operators for this database type | |
225 | ASSERT_EQ(0, db->create_and_open(cout)); | |
226 | { | |
227 | KeyValueDB::Transaction t = db->get_transaction(); | |
228 | bufferlist v1, v2, v3; | |
229 | v1.append(string("1")); | |
230 | v2.append(string("2")); | |
231 | v3.append(string("3")); | |
232 | t->set("P", "K1", v1); | |
233 | t->set("A", "A1", v2); | |
234 | t->rmkey("A", "A2"); | |
235 | t->merge("A", "A2", v3); | |
236 | db->submit_transaction_sync(t); | |
237 | } | |
238 | { | |
239 | bufferlist v1, v2, v3; | |
240 | ASSERT_EQ(0, db->get("P", "K1", &v1)); | |
241 | ASSERT_EQ(tostr(v1), "1"); | |
242 | ASSERT_EQ(0, db->get("A", "A1", &v2)); | |
243 | ASSERT_EQ(tostr(v2), "2"); | |
244 | ASSERT_EQ(0, db->get("A", "A2", &v3)); | |
245 | ASSERT_EQ(tostr(v3), "?3"); | |
246 | } | |
247 | { | |
248 | KeyValueDB::Transaction t = db->get_transaction(); | |
249 | bufferlist v1; | |
250 | v1.append(string("1")); | |
251 | t->merge("A", "A2", v1); | |
252 | db->submit_transaction_sync(t); | |
253 | } | |
254 | { | |
255 | bufferlist v; | |
256 | ASSERT_EQ(0, db->get("A", "A2", &v)); | |
257 | ASSERT_EQ(tostr(v), "?31"); | |
258 | } | |
259 | fini(); | |
260 | } | |
261 | ||
262 | TEST_P(KVTest, RMRange) { | |
263 | ASSERT_EQ(0, db->create_and_open(cout)); | |
264 | bufferlist value; | |
265 | value.append("value"); | |
266 | { | |
267 | KeyValueDB::Transaction t = db->get_transaction(); | |
268 | t->set("prefix", "key1", value); | |
269 | t->set("prefix", "key2", value); | |
270 | t->set("prefix", "key3", value); | |
271 | t->set("prefix", "key4", value); | |
272 | t->set("prefix", "key45", value); | |
273 | t->set("prefix", "key5", value); | |
274 | t->set("prefix", "key6", value); | |
275 | db->submit_transaction_sync(t); | |
276 | } | |
277 | ||
278 | { | |
279 | KeyValueDB::Transaction t = db->get_transaction(); | |
280 | t->set("prefix", "key7", value); | |
281 | t->set("prefix", "key8", value); | |
282 | t->rm_range_keys("prefix", "key2", "key7"); | |
283 | db->submit_transaction_sync(t); | |
284 | bufferlist v1, v2; | |
285 | ASSERT_EQ(0, db->get("prefix", "key1", &v1)); | |
286 | v1.clear(); | |
287 | ASSERT_EQ(-ENOENT, db->get("prefix", "key45", &v1)); | |
288 | ASSERT_EQ(0, db->get("prefix", "key8", &v1)); | |
289 | v1.clear(); | |
290 | ASSERT_EQ(-ENOENT, db->get("prefix", "key2", &v1)); | |
291 | ASSERT_EQ(0, db->get("prefix", "key7", &v2)); | |
292 | } | |
293 | ||
294 | { | |
295 | KeyValueDB::Transaction t = db->get_transaction(); | |
296 | t->rm_range_keys("prefix", "key", "key"); | |
297 | db->submit_transaction_sync(t); | |
298 | bufferlist v1, v2; | |
299 | ASSERT_EQ(0, db->get("prefix", "key1", &v1)); | |
300 | ASSERT_EQ(0, db->get("prefix", "key8", &v2)); | |
301 | } | |
302 | ||
303 | { | |
304 | KeyValueDB::Transaction t = db->get_transaction(); | |
305 | t->rm_range_keys("prefix", "key-", "key~"); | |
306 | db->submit_transaction_sync(t); | |
307 | bufferlist v1, v2; | |
308 | ASSERT_EQ(-ENOENT, db->get("prefix", "key1", &v1)); | |
309 | ASSERT_EQ(-ENOENT, db->get("prefix", "key8", &v2)); | |
310 | } | |
311 | ||
312 | fini(); | |
313 | } | |
314 | ||
315 | TEST_P(KVTest, RocksDBColumnFamilyTest) { | |
316 | if(string(GetParam()) != "rocksdb") | |
317 | return; | |
318 | ||
319 | std::vector<KeyValueDB::ColumnFamily> cfs; | |
320 | cfs.push_back(KeyValueDB::ColumnFamily("cf1", "")); | |
321 | cfs.push_back(KeyValueDB::ColumnFamily("cf2", "")); | |
322 | ASSERT_EQ(0, db->init(g_conf()->bluestore_rocksdb_options)); | |
323 | cout << "creating two column families and opening them" << std::endl; | |
324 | ASSERT_EQ(0, db->create_and_open(cout, cfs)); | |
325 | { | |
326 | KeyValueDB::Transaction t = db->get_transaction(); | |
327 | bufferlist value; | |
328 | value.append("value"); | |
329 | cout << "write a transaction includes three keys in different CFs" << std::endl; | |
330 | t->set("prefix", "key", value); | |
331 | t->set("cf1", "key", value); | |
332 | t->set("cf2", "key2", value); | |
333 | ASSERT_EQ(0, db->submit_transaction_sync(t)); | |
334 | } | |
335 | fini(); | |
336 | ||
337 | init(); | |
338 | ASSERT_EQ(0, db->open(cout, cfs)); | |
339 | { | |
340 | bufferlist v1, v2, v3; | |
341 | cout << "reopen db and read those keys" << std::endl; | |
342 | ASSERT_EQ(0, db->get("prefix", "key", &v1)); | |
343 | ASSERT_EQ(0, _bl_to_str(v1) != "value"); | |
344 | ASSERT_EQ(0, db->get("cf1", "key", &v2)); | |
345 | ASSERT_EQ(0, _bl_to_str(v2) != "value"); | |
346 | ASSERT_EQ(0, db->get("cf2", "key2", &v3)); | |
347 | ASSERT_EQ(0, _bl_to_str(v2) != "value"); | |
348 | } | |
349 | { | |
350 | cout << "delete two keys in CFs" << std::endl; | |
351 | KeyValueDB::Transaction t = db->get_transaction(); | |
352 | t->rmkey("prefix", "key"); | |
353 | t->rmkey("cf2", "key2"); | |
354 | ASSERT_EQ(0, db->submit_transaction_sync(t)); | |
355 | } | |
356 | fini(); | |
357 | ||
358 | init(); | |
359 | ASSERT_EQ(0, db->open(cout, cfs)); | |
360 | { | |
361 | cout << "reopen db and read keys again." << std::endl; | |
362 | bufferlist v1, v2, v3; | |
363 | ASSERT_EQ(-ENOENT, db->get("prefix", "key", &v1)); | |
364 | ASSERT_EQ(0, db->get("cf1", "key", &v2)); | |
365 | ASSERT_EQ(0, _bl_to_str(v2) != "value"); | |
366 | ASSERT_EQ(-ENOENT, db->get("cf2", "key2", &v3)); | |
367 | } | |
368 | fini(); | |
369 | } | |
370 | ||
371 | TEST_P(KVTest, RocksDBIteratorTest) { | |
372 | if(string(GetParam()) != "rocksdb") | |
373 | return; | |
374 | ||
375 | std::vector<KeyValueDB::ColumnFamily> cfs; | |
376 | cfs.push_back(KeyValueDB::ColumnFamily("cf1", "")); | |
377 | ASSERT_EQ(0, db->init(g_conf()->bluestore_rocksdb_options)); | |
378 | cout << "creating one column family and opening it" << std::endl; | |
379 | ASSERT_EQ(0, db->create_and_open(cout, cfs)); | |
380 | { | |
381 | KeyValueDB::Transaction t = db->get_transaction(); | |
382 | bufferlist bl1; | |
383 | bl1.append("hello"); | |
384 | bufferlist bl2; | |
385 | bl2.append("world"); | |
386 | cout << "write some kv pairs into default and new CFs" << std::endl; | |
387 | t->set("prefix", "key1", bl1); | |
388 | t->set("prefix", "key2", bl2); | |
389 | t->set("cf1", "key1", bl1); | |
390 | t->set("cf1", "key2", bl2); | |
391 | ASSERT_EQ(0, db->submit_transaction_sync(t)); | |
392 | } | |
393 | { | |
394 | cout << "iterating the default CF" << std::endl; | |
395 | KeyValueDB::Iterator iter = db->get_iterator("prefix"); | |
396 | iter->seek_to_first(); | |
397 | ASSERT_EQ(1, iter->valid()); | |
398 | ASSERT_EQ("key1", iter->key()); | |
399 | ASSERT_EQ("hello", _bl_to_str(iter->value())); | |
400 | ASSERT_EQ(0, iter->next()); | |
401 | ASSERT_EQ(1, iter->valid()); | |
402 | ASSERT_EQ("key2", iter->key()); | |
403 | ASSERT_EQ("world", _bl_to_str(iter->value())); | |
404 | } | |
405 | { | |
406 | cout << "iterating the new CF" << std::endl; | |
407 | KeyValueDB::Iterator iter = db->get_iterator("cf1"); | |
408 | iter->seek_to_first(); | |
409 | ASSERT_EQ(1, iter->valid()); | |
410 | ASSERT_EQ("key1", iter->key()); | |
411 | ASSERT_EQ("hello", _bl_to_str(iter->value())); | |
412 | ASSERT_EQ(0, iter->next()); | |
413 | ASSERT_EQ(1, iter->valid()); | |
414 | ASSERT_EQ("key2", iter->key()); | |
415 | ASSERT_EQ("world", _bl_to_str(iter->value())); | |
416 | } | |
417 | fini(); | |
418 | } | |
419 | ||
420 | TEST_P(KVTest, RocksDBCFMerge) { | |
421 | if(string(GetParam()) != "rocksdb") | |
422 | return; | |
423 | ||
424 | shared_ptr<KeyValueDB::MergeOperator> p(new AppendMOP); | |
425 | int r = db->set_merge_operator("cf1",p); | |
426 | if (r < 0) | |
427 | return; // No merge operators for this database type | |
428 | std::vector<KeyValueDB::ColumnFamily> cfs; | |
429 | cfs.push_back(KeyValueDB::ColumnFamily("cf1", "")); | |
430 | ASSERT_EQ(0, db->init(g_conf()->bluestore_rocksdb_options)); | |
431 | cout << "creating one column family and opening it" << std::endl; | |
432 | ASSERT_EQ(0, db->create_and_open(cout, cfs)); | |
433 | ||
434 | { | |
435 | KeyValueDB::Transaction t = db->get_transaction(); | |
436 | bufferlist v1, v2, v3; | |
437 | v1.append(string("1")); | |
438 | v2.append(string("2")); | |
439 | v3.append(string("3")); | |
440 | t->set("P", "K1", v1); | |
441 | t->set("cf1", "A1", v2); | |
442 | t->rmkey("cf1", "A2"); | |
443 | t->merge("cf1", "A2", v3); | |
444 | db->submit_transaction_sync(t); | |
445 | } | |
446 | { | |
447 | bufferlist v1, v2, v3; | |
448 | ASSERT_EQ(0, db->get("P", "K1", &v1)); | |
449 | ASSERT_EQ(tostr(v1), "1"); | |
450 | ASSERT_EQ(0, db->get("cf1", "A1", &v2)); | |
451 | ASSERT_EQ(tostr(v2), "2"); | |
452 | ASSERT_EQ(0, db->get("cf1", "A2", &v3)); | |
453 | ASSERT_EQ(tostr(v3), "?3"); | |
454 | } | |
455 | { | |
456 | KeyValueDB::Transaction t = db->get_transaction(); | |
457 | bufferlist v1; | |
458 | v1.append(string("1")); | |
459 | t->merge("cf1", "A2", v1); | |
460 | db->submit_transaction_sync(t); | |
461 | } | |
462 | { | |
463 | bufferlist v; | |
464 | ASSERT_EQ(0, db->get("cf1", "A2", &v)); | |
465 | ASSERT_EQ(tostr(v), "?31"); | |
466 | } | |
467 | fini(); | |
468 | } | |
469 | ||
470 | INSTANTIATE_TEST_CASE_P( | |
471 | KeyValueDB, | |
472 | KVTest, | |
473 | ::testing::Values("leveldb", "rocksdb", "memdb")); | |
474 | ||
475 | #else | |
476 | ||
477 | // Google Test may not support value-parameterized tests with some | |
478 | // compilers. If we use conditional compilation to compile out all | |
479 | // code referring to the gtest_main library, MSVC linker will not link | |
480 | // that library at all and consequently complain about missing entry | |
481 | // point defined in that library (fatal error LNK1561: entry point | |
482 | // must be defined). This dummy test keeps gtest_main linked in. | |
483 | TEST(DummyTest, ValueParameterizedTestsAreNotSupportedOnThisPlatform) {} | |
484 | ||
485 | #endif | |
486 | ||
487 | int main(int argc, char **argv) { | |
488 | vector<const char*> args; | |
489 | argv_to_vec(argc, (const char **)argv, args); | |
490 | ||
491 | auto cct = global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, | |
492 | CODE_ENVIRONMENT_UTILITY, | |
493 | CINIT_FLAG_NO_DEFAULT_CONFIG_FILE); | |
494 | common_init_finish(g_ceph_context); | |
495 | g_ceph_context->_conf.set_val( | |
496 | "enable_experimental_unrecoverable_data_corrupting_features", | |
497 | "rocksdb, memdb"); | |
498 | g_ceph_context->_conf.apply_changes(nullptr); | |
499 | ||
500 | ::testing::InitGoogleTest(&argc, argv); | |
501 | return RUN_ALL_TESTS(); | |
502 | } |