]>
Commit | Line | Data |
---|---|---|
7c673cae | 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. |
11fdf7f2 TL |
2 | // This source code is licensed under both the GPLv2 (found in the |
3 | // COPYING file in the root directory) and Apache 2.0 License | |
4 | // (found in the LICENSE.Apache file in the root directory). | |
7c673cae FG |
5 | |
6 | #ifndef ROCKSDB_LITE | |
7 | ||
f67539c2 | 8 | #include "db/db_impl/db_impl.h" |
7c673cae FG |
9 | #include "rocksdb/db.h" |
10 | #include "rocksdb/env.h" | |
f67539c2 TL |
11 | #include "table/cuckoo/cuckoo_table_factory.h" |
12 | #include "table/cuckoo/cuckoo_table_reader.h" | |
7c673cae | 13 | #include "table/meta_blocks.h" |
f67539c2 TL |
14 | #include "test_util/testharness.h" |
15 | #include "test_util/testutil.h" | |
20effc67 | 16 | #include "util/cast_util.h" |
7c673cae | 17 | #include "util/string_util.h" |
7c673cae | 18 | |
f67539c2 | 19 | namespace ROCKSDB_NAMESPACE { |
7c673cae FG |
20 | |
21 | class CuckooTableDBTest : public testing::Test { | |
22 | private: | |
23 | std::string dbname_; | |
24 | Env* env_; | |
25 | DB* db_; | |
26 | ||
27 | public: | |
28 | CuckooTableDBTest() : env_(Env::Default()) { | |
11fdf7f2 | 29 | dbname_ = test::PerThreadDBPath("cuckoo_table_db_test"); |
7c673cae FG |
30 | EXPECT_OK(DestroyDB(dbname_, Options())); |
31 | db_ = nullptr; | |
32 | Reopen(); | |
33 | } | |
34 | ||
494da23a | 35 | ~CuckooTableDBTest() override { |
7c673cae FG |
36 | delete db_; |
37 | EXPECT_OK(DestroyDB(dbname_, Options())); | |
38 | } | |
39 | ||
40 | Options CurrentOptions() { | |
41 | Options options; | |
42 | options.table_factory.reset(NewCuckooTableFactory()); | |
43 | options.memtable_factory.reset(NewHashLinkListRepFactory(4, 0, 3, true)); | |
44 | options.allow_mmap_reads = true; | |
45 | options.create_if_missing = true; | |
46 | options.allow_concurrent_memtable_write = false; | |
47 | return options; | |
48 | } | |
49 | ||
20effc67 | 50 | DBImpl* dbfull() { return static_cast_with_check<DBImpl>(db_); } |
7c673cae FG |
51 | |
52 | // The following util methods are copied from plain_table_db_test. | |
53 | void Reopen(Options* options = nullptr) { | |
54 | delete db_; | |
55 | db_ = nullptr; | |
56 | Options opts; | |
57 | if (options != nullptr) { | |
58 | opts = *options; | |
59 | } else { | |
60 | opts = CurrentOptions(); | |
61 | opts.create_if_missing = true; | |
62 | } | |
63 | ASSERT_OK(DB::Open(opts, dbname_, &db_)); | |
64 | } | |
65 | ||
20effc67 TL |
66 | void DestroyAndReopen(Options* options) { |
67 | assert(options); | |
68 | ASSERT_OK(db_->Close()); | |
69 | delete db_; | |
70 | db_ = nullptr; | |
71 | ASSERT_OK(DestroyDB(dbname_, *options)); | |
72 | Reopen(options); | |
73 | } | |
74 | ||
7c673cae FG |
75 | Status Put(const Slice& k, const Slice& v) { |
76 | return db_->Put(WriteOptions(), k, v); | |
77 | } | |
78 | ||
79 | Status Delete(const std::string& k) { | |
80 | return db_->Delete(WriteOptions(), k); | |
81 | } | |
82 | ||
83 | std::string Get(const std::string& k) { | |
84 | ReadOptions options; | |
85 | std::string result; | |
86 | Status s = db_->Get(options, k, &result); | |
87 | if (s.IsNotFound()) { | |
88 | result = "NOT_FOUND"; | |
89 | } else if (!s.ok()) { | |
90 | result = s.ToString(); | |
91 | } | |
92 | return result; | |
93 | } | |
94 | ||
95 | int NumTableFilesAtLevel(int level) { | |
96 | std::string property; | |
97 | EXPECT_TRUE(db_->GetProperty( | |
98 | "rocksdb.num-files-at-level" + NumberToString(level), &property)); | |
99 | return atoi(property.c_str()); | |
100 | } | |
101 | ||
102 | // Return spread of files per level | |
103 | std::string FilesPerLevel() { | |
104 | std::string result; | |
105 | size_t last_non_zero_offset = 0; | |
106 | for (int level = 0; level < db_->NumberLevels(); level++) { | |
107 | int f = NumTableFilesAtLevel(level); | |
108 | char buf[100]; | |
109 | snprintf(buf, sizeof(buf), "%s%d", (level ? "," : ""), f); | |
110 | result += buf; | |
111 | if (f > 0) { | |
112 | last_non_zero_offset = result.size(); | |
113 | } | |
114 | } | |
115 | result.resize(last_non_zero_offset); | |
116 | return result; | |
117 | } | |
118 | }; | |
119 | ||
120 | TEST_F(CuckooTableDBTest, Flush) { | |
121 | // Try with empty DB first. | |
122 | ASSERT_TRUE(dbfull() != nullptr); | |
123 | ASSERT_EQ("NOT_FOUND", Get("key2")); | |
124 | ||
125 | // Add some values to db. | |
126 | Options options = CurrentOptions(); | |
127 | Reopen(&options); | |
128 | ||
129 | ASSERT_OK(Put("key1", "v1")); | |
130 | ASSERT_OK(Put("key2", "v2")); | |
131 | ASSERT_OK(Put("key3", "v3")); | |
132 | dbfull()->TEST_FlushMemTable(); | |
133 | ||
134 | TablePropertiesCollection ptc; | |
135 | reinterpret_cast<DB*>(dbfull())->GetPropertiesOfAllTables(&ptc); | |
136 | ASSERT_EQ(1U, ptc.size()); | |
137 | ASSERT_EQ(3U, ptc.begin()->second->num_entries); | |
138 | ASSERT_EQ("1", FilesPerLevel()); | |
139 | ||
140 | ASSERT_EQ("v1", Get("key1")); | |
141 | ASSERT_EQ("v2", Get("key2")); | |
142 | ASSERT_EQ("v3", Get("key3")); | |
143 | ASSERT_EQ("NOT_FOUND", Get("key4")); | |
144 | ||
145 | // Now add more keys and flush. | |
146 | ASSERT_OK(Put("key4", "v4")); | |
147 | ASSERT_OK(Put("key5", "v5")); | |
148 | ASSERT_OK(Put("key6", "v6")); | |
149 | dbfull()->TEST_FlushMemTable(); | |
150 | ||
151 | reinterpret_cast<DB*>(dbfull())->GetPropertiesOfAllTables(&ptc); | |
152 | ASSERT_EQ(2U, ptc.size()); | |
153 | auto row = ptc.begin(); | |
154 | ASSERT_EQ(3U, row->second->num_entries); | |
155 | ASSERT_EQ(3U, (++row)->second->num_entries); | |
156 | ASSERT_EQ("2", FilesPerLevel()); | |
157 | ASSERT_EQ("v1", Get("key1")); | |
158 | ASSERT_EQ("v2", Get("key2")); | |
159 | ASSERT_EQ("v3", Get("key3")); | |
160 | ASSERT_EQ("v4", Get("key4")); | |
161 | ASSERT_EQ("v5", Get("key5")); | |
162 | ASSERT_EQ("v6", Get("key6")); | |
163 | ||
164 | ASSERT_OK(Delete("key6")); | |
165 | ASSERT_OK(Delete("key5")); | |
166 | ASSERT_OK(Delete("key4")); | |
167 | dbfull()->TEST_FlushMemTable(); | |
168 | reinterpret_cast<DB*>(dbfull())->GetPropertiesOfAllTables(&ptc); | |
169 | ASSERT_EQ(3U, ptc.size()); | |
170 | row = ptc.begin(); | |
171 | ASSERT_EQ(3U, row->second->num_entries); | |
172 | ASSERT_EQ(3U, (++row)->second->num_entries); | |
173 | ASSERT_EQ(3U, (++row)->second->num_entries); | |
174 | ASSERT_EQ("3", FilesPerLevel()); | |
175 | ASSERT_EQ("v1", Get("key1")); | |
176 | ASSERT_EQ("v2", Get("key2")); | |
177 | ASSERT_EQ("v3", Get("key3")); | |
178 | ASSERT_EQ("NOT_FOUND", Get("key4")); | |
179 | ASSERT_EQ("NOT_FOUND", Get("key5")); | |
180 | ASSERT_EQ("NOT_FOUND", Get("key6")); | |
181 | } | |
182 | ||
183 | TEST_F(CuckooTableDBTest, FlushWithDuplicateKeys) { | |
184 | Options options = CurrentOptions(); | |
185 | Reopen(&options); | |
186 | ASSERT_OK(Put("key1", "v1")); | |
187 | ASSERT_OK(Put("key2", "v2")); | |
188 | ASSERT_OK(Put("key1", "v3")); // Duplicate | |
189 | dbfull()->TEST_FlushMemTable(); | |
190 | ||
191 | TablePropertiesCollection ptc; | |
192 | reinterpret_cast<DB*>(dbfull())->GetPropertiesOfAllTables(&ptc); | |
193 | ASSERT_EQ(1U, ptc.size()); | |
194 | ASSERT_EQ(2U, ptc.begin()->second->num_entries); | |
195 | ASSERT_EQ("1", FilesPerLevel()); | |
196 | ASSERT_EQ("v3", Get("key1")); | |
197 | ASSERT_EQ("v2", Get("key2")); | |
198 | } | |
199 | ||
200 | namespace { | |
201 | static std::string Key(int i) { | |
202 | char buf[100]; | |
203 | snprintf(buf, sizeof(buf), "key_______%06d", i); | |
204 | return std::string(buf); | |
205 | } | |
206 | static std::string Uint64Key(uint64_t i) { | |
207 | std::string str; | |
208 | str.resize(8); | |
209 | memcpy(&str[0], static_cast<void*>(&i), 8); | |
210 | return str; | |
211 | } | |
212 | } // namespace. | |
213 | ||
214 | TEST_F(CuckooTableDBTest, Uint64Comparator) { | |
215 | Options options = CurrentOptions(); | |
216 | options.comparator = test::Uint64Comparator(); | |
20effc67 | 217 | DestroyAndReopen(&options); |
7c673cae FG |
218 | |
219 | ASSERT_OK(Put(Uint64Key(1), "v1")); | |
220 | ASSERT_OK(Put(Uint64Key(2), "v2")); | |
221 | ASSERT_OK(Put(Uint64Key(3), "v3")); | |
222 | dbfull()->TEST_FlushMemTable(); | |
223 | ||
224 | ASSERT_EQ("v1", Get(Uint64Key(1))); | |
225 | ASSERT_EQ("v2", Get(Uint64Key(2))); | |
226 | ASSERT_EQ("v3", Get(Uint64Key(3))); | |
227 | ASSERT_EQ("NOT_FOUND", Get(Uint64Key(4))); | |
228 | ||
229 | // Add more keys. | |
230 | ASSERT_OK(Delete(Uint64Key(2))); // Delete. | |
231 | dbfull()->TEST_FlushMemTable(); | |
232 | ASSERT_OK(Put(Uint64Key(3), "v0")); // Update. | |
233 | ASSERT_OK(Put(Uint64Key(4), "v4")); | |
234 | dbfull()->TEST_FlushMemTable(); | |
235 | ASSERT_EQ("v1", Get(Uint64Key(1))); | |
236 | ASSERT_EQ("NOT_FOUND", Get(Uint64Key(2))); | |
237 | ASSERT_EQ("v0", Get(Uint64Key(3))); | |
238 | ASSERT_EQ("v4", Get(Uint64Key(4))); | |
239 | } | |
240 | ||
241 | TEST_F(CuckooTableDBTest, CompactionIntoMultipleFiles) { | |
242 | // Create a big L0 file and check it compacts into multiple files in L1. | |
243 | Options options = CurrentOptions(); | |
244 | options.write_buffer_size = 270 << 10; | |
245 | // Two SST files should be created, each containing 14 keys. | |
246 | // Number of buckets will be 16. Total size ~156 KB. | |
247 | options.target_file_size_base = 160 << 10; | |
248 | Reopen(&options); | |
249 | ||
250 | // Write 28 values, each 10016 B ~ 10KB | |
251 | for (int idx = 0; idx < 28; ++idx) { | |
11fdf7f2 | 252 | ASSERT_OK(Put(Key(idx), std::string(10000, 'a' + char(idx)))); |
7c673cae FG |
253 | } |
254 | dbfull()->TEST_WaitForFlushMemTable(); | |
255 | ASSERT_EQ("1", FilesPerLevel()); | |
256 | ||
257 | dbfull()->TEST_CompactRange(0, nullptr, nullptr, nullptr, | |
258 | true /* disallow trivial move */); | |
259 | ASSERT_EQ("0,2", FilesPerLevel()); | |
260 | for (int idx = 0; idx < 28; ++idx) { | |
11fdf7f2 | 261 | ASSERT_EQ(std::string(10000, 'a' + char(idx)), Get(Key(idx))); |
7c673cae FG |
262 | } |
263 | } | |
264 | ||
265 | TEST_F(CuckooTableDBTest, SameKeyInsertedInTwoDifferentFilesAndCompacted) { | |
266 | // Insert same key twice so that they go to different SST files. Then wait for | |
267 | // compaction and check if the latest value is stored and old value removed. | |
268 | Options options = CurrentOptions(); | |
269 | options.write_buffer_size = 100 << 10; // 100KB | |
270 | options.level0_file_num_compaction_trigger = 2; | |
271 | Reopen(&options); | |
272 | ||
273 | // Write 11 values, each 10016 B | |
274 | for (int idx = 0; idx < 11; ++idx) { | |
275 | ASSERT_OK(Put(Key(idx), std::string(10000, 'a'))); | |
276 | } | |
277 | dbfull()->TEST_WaitForFlushMemTable(); | |
278 | ASSERT_EQ("1", FilesPerLevel()); | |
279 | ||
280 | // Generate one more file in level-0, and should trigger level-0 compaction | |
281 | for (int idx = 0; idx < 11; ++idx) { | |
11fdf7f2 | 282 | ASSERT_OK(Put(Key(idx), std::string(10000, 'a' + char(idx)))); |
7c673cae FG |
283 | } |
284 | dbfull()->TEST_WaitForFlushMemTable(); | |
285 | dbfull()->TEST_CompactRange(0, nullptr, nullptr); | |
286 | ||
287 | ASSERT_EQ("0,1", FilesPerLevel()); | |
288 | for (int idx = 0; idx < 11; ++idx) { | |
11fdf7f2 | 289 | ASSERT_EQ(std::string(10000, 'a' + char(idx)), Get(Key(idx))); |
7c673cae FG |
290 | } |
291 | } | |
292 | ||
293 | TEST_F(CuckooTableDBTest, AdaptiveTable) { | |
294 | Options options = CurrentOptions(); | |
295 | ||
f67539c2 TL |
296 | // Ensure options compatible with PlainTable |
297 | options.prefix_extractor.reset(NewCappedPrefixTransform(8)); | |
298 | ||
7c673cae FG |
299 | // Write some keys using cuckoo table. |
300 | options.table_factory.reset(NewCuckooTableFactory()); | |
301 | Reopen(&options); | |
302 | ||
303 | ASSERT_OK(Put("key1", "v1")); | |
304 | ASSERT_OK(Put("key2", "v2")); | |
305 | ASSERT_OK(Put("key3", "v3")); | |
306 | dbfull()->TEST_FlushMemTable(); | |
307 | ||
308 | // Write some keys using plain table. | |
f67539c2 TL |
309 | std::shared_ptr<TableFactory> block_based_factory( |
310 | NewBlockBasedTableFactory()); | |
311 | std::shared_ptr<TableFactory> plain_table_factory( | |
312 | NewPlainTableFactory()); | |
313 | std::shared_ptr<TableFactory> cuckoo_table_factory( | |
314 | NewCuckooTableFactory()); | |
7c673cae | 315 | options.create_if_missing = false; |
f67539c2 TL |
316 | options.table_factory.reset(NewAdaptiveTableFactory( |
317 | plain_table_factory, block_based_factory, plain_table_factory, | |
318 | cuckoo_table_factory)); | |
7c673cae FG |
319 | Reopen(&options); |
320 | ASSERT_OK(Put("key4", "v4")); | |
321 | ASSERT_OK(Put("key1", "v5")); | |
322 | dbfull()->TEST_FlushMemTable(); | |
323 | ||
324 | // Write some keys using block based table. | |
f67539c2 TL |
325 | options.table_factory.reset(NewAdaptiveTableFactory( |
326 | block_based_factory, block_based_factory, plain_table_factory, | |
327 | cuckoo_table_factory)); | |
7c673cae FG |
328 | Reopen(&options); |
329 | ASSERT_OK(Put("key5", "v6")); | |
330 | ASSERT_OK(Put("key2", "v7")); | |
331 | dbfull()->TEST_FlushMemTable(); | |
332 | ||
333 | ASSERT_EQ("v5", Get("key1")); | |
334 | ASSERT_EQ("v7", Get("key2")); | |
335 | ASSERT_EQ("v3", Get("key3")); | |
336 | ASSERT_EQ("v4", Get("key4")); | |
337 | ASSERT_EQ("v6", Get("key5")); | |
338 | } | |
f67539c2 | 339 | } // namespace ROCKSDB_NAMESPACE |
7c673cae FG |
340 | |
341 | int main(int argc, char** argv) { | |
f67539c2 | 342 | if (ROCKSDB_NAMESPACE::port::kLittleEndian) { |
7c673cae FG |
343 | ::testing::InitGoogleTest(&argc, argv); |
344 | return RUN_ALL_TESTS(); | |
f67539c2 | 345 | } else { |
7c673cae FG |
346 | fprintf(stderr, "SKIPPED as Cuckoo table doesn't support Big Endian\n"); |
347 | return 0; | |
348 | } | |
349 | } | |
350 | ||
351 | #else | |
352 | #include <stdio.h> | |
353 | ||
11fdf7f2 | 354 | int main(int /*argc*/, char** /*argv*/) { |
7c673cae FG |
355 | fprintf(stderr, "SKIPPED as Cuckoo table is not supported in ROCKSDB_LITE\n"); |
356 | return 0; | |
357 | } | |
358 | ||
359 | #endif // ROCKSDB_LITE |