1 // Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
2 // This source code is licensed under both the GPLv2 (found in the
3 // COPYING file in the root directory) and Apache 2.0 License
4 // (found in the LICENSE.Apache file in the root directory).
8 #include "db/db_impl/db_impl.h"
9 #include "db/db_test_util.h"
10 #include "rocksdb/db.h"
11 #include "rocksdb/env.h"
12 #include "table/cuckoo/cuckoo_table_factory.h"
13 #include "table/cuckoo/cuckoo_table_reader.h"
14 #include "table/meta_blocks.h"
15 #include "test_util/testharness.h"
16 #include "test_util/testutil.h"
17 #include "util/cast_util.h"
18 #include "util/string_util.h"
20 namespace ROCKSDB_NAMESPACE
{
22 class CuckooTableDBTest
: public testing::Test
{
29 CuckooTableDBTest() : env_(Env::Default()) {
30 dbname_
= test::PerThreadDBPath("cuckoo_table_db_test");
31 EXPECT_OK(DestroyDB(dbname_
, Options()));
36 ~CuckooTableDBTest() override
{
38 EXPECT_OK(DestroyDB(dbname_
, Options()));
41 Options
CurrentOptions() {
43 options
.table_factory
.reset(NewCuckooTableFactory());
44 options
.memtable_factory
.reset(NewHashLinkListRepFactory(4, 0, 3, true));
45 options
.allow_mmap_reads
= true;
46 options
.create_if_missing
= true;
47 options
.allow_concurrent_memtable_write
= false;
51 DBImpl
* dbfull() { return static_cast_with_check
<DBImpl
>(db_
); }
53 // The following util methods are copied from plain_table_db_test.
54 void Reopen(Options
* options
= nullptr) {
58 if (options
!= nullptr) {
61 opts
= CurrentOptions();
62 opts
.create_if_missing
= true;
64 ASSERT_OK(DB::Open(opts
, dbname_
, &db_
));
67 void DestroyAndReopen(Options
* options
) {
69 ASSERT_OK(db_
->Close());
72 ASSERT_OK(DestroyDB(dbname_
, *options
));
76 Status
Put(const Slice
& k
, const Slice
& v
) {
77 return db_
->Put(WriteOptions(), k
, v
);
80 Status
Delete(const std::string
& k
) { return db_
->Delete(WriteOptions(), k
); }
82 std::string
Get(const std::string
& k
) {
85 Status s
= db_
->Get(options
, k
, &result
);
89 result
= s
.ToString();
94 int NumTableFilesAtLevel(int level
) {
96 EXPECT_TRUE(db_
->GetProperty(
97 "rocksdb.num-files-at-level" + std::to_string(level
), &property
));
98 return atoi(property
.c_str());
101 // Return spread of files per level
102 std::string
FilesPerLevel() {
104 size_t last_non_zero_offset
= 0;
105 for (int level
= 0; level
< db_
->NumberLevels(); level
++) {
106 int f
= NumTableFilesAtLevel(level
);
108 snprintf(buf
, sizeof(buf
), "%s%d", (level
? "," : ""), f
);
111 last_non_zero_offset
= result
.size();
114 result
.resize(last_non_zero_offset
);
119 TEST_F(CuckooTableDBTest
, Flush
) {
120 // Try with empty DB first.
121 ASSERT_TRUE(dbfull() != nullptr);
122 ASSERT_EQ("NOT_FOUND", Get("key2"));
124 // Add some values to db.
125 Options options
= CurrentOptions();
128 ASSERT_OK(Put("key1", "v1"));
129 ASSERT_OK(Put("key2", "v2"));
130 ASSERT_OK(Put("key3", "v3"));
131 ASSERT_OK(dbfull()->TEST_FlushMemTable());
133 TablePropertiesCollection ptc
;
134 ASSERT_OK(reinterpret_cast<DB
*>(dbfull())->GetPropertiesOfAllTables(&ptc
));
135 VerifySstUniqueIds(ptc
);
136 ASSERT_EQ(1U, ptc
.size());
137 ASSERT_EQ(3U, ptc
.begin()->second
->num_entries
);
138 ASSERT_EQ("1", FilesPerLevel());
140 ASSERT_EQ("v1", Get("key1"));
141 ASSERT_EQ("v2", Get("key2"));
142 ASSERT_EQ("v3", Get("key3"));
143 ASSERT_EQ("NOT_FOUND", Get("key4"));
145 // Now add more keys and flush.
146 ASSERT_OK(Put("key4", "v4"));
147 ASSERT_OK(Put("key5", "v5"));
148 ASSERT_OK(Put("key6", "v6"));
149 ASSERT_OK(dbfull()->TEST_FlushMemTable());
151 ASSERT_OK(reinterpret_cast<DB
*>(dbfull())->GetPropertiesOfAllTables(&ptc
));
152 VerifySstUniqueIds(ptc
);
153 ASSERT_EQ(2U, ptc
.size());
154 auto row
= ptc
.begin();
155 ASSERT_EQ(3U, row
->second
->num_entries
);
156 ASSERT_EQ(3U, (++row
)->second
->num_entries
);
157 ASSERT_EQ("2", FilesPerLevel());
158 ASSERT_EQ("v1", Get("key1"));
159 ASSERT_EQ("v2", Get("key2"));
160 ASSERT_EQ("v3", Get("key3"));
161 ASSERT_EQ("v4", Get("key4"));
162 ASSERT_EQ("v5", Get("key5"));
163 ASSERT_EQ("v6", Get("key6"));
165 ASSERT_OK(Delete("key6"));
166 ASSERT_OK(Delete("key5"));
167 ASSERT_OK(Delete("key4"));
168 ASSERT_OK(dbfull()->TEST_FlushMemTable());
169 ASSERT_OK(reinterpret_cast<DB
*>(dbfull())->GetPropertiesOfAllTables(&ptc
));
170 VerifySstUniqueIds(ptc
);
171 ASSERT_EQ(3U, ptc
.size());
173 ASSERT_EQ(3U, row
->second
->num_entries
);
174 ASSERT_EQ(3U, (++row
)->second
->num_entries
);
175 ASSERT_EQ(3U, (++row
)->second
->num_entries
);
176 ASSERT_EQ("3", FilesPerLevel());
177 ASSERT_EQ("v1", Get("key1"));
178 ASSERT_EQ("v2", Get("key2"));
179 ASSERT_EQ("v3", Get("key3"));
180 ASSERT_EQ("NOT_FOUND", Get("key4"));
181 ASSERT_EQ("NOT_FOUND", Get("key5"));
182 ASSERT_EQ("NOT_FOUND", Get("key6"));
185 TEST_F(CuckooTableDBTest
, FlushWithDuplicateKeys
) {
186 Options options
= CurrentOptions();
188 ASSERT_OK(Put("key1", "v1"));
189 ASSERT_OK(Put("key2", "v2"));
190 ASSERT_OK(Put("key1", "v3")); // Duplicate
191 ASSERT_OK(dbfull()->TEST_FlushMemTable());
193 TablePropertiesCollection ptc
;
194 ASSERT_OK(reinterpret_cast<DB
*>(dbfull())->GetPropertiesOfAllTables(&ptc
));
195 VerifySstUniqueIds(ptc
);
196 ASSERT_EQ(1U, ptc
.size());
197 ASSERT_EQ(2U, ptc
.begin()->second
->num_entries
);
198 ASSERT_EQ("1", FilesPerLevel());
199 ASSERT_EQ("v3", Get("key1"));
200 ASSERT_EQ("v2", Get("key2"));
204 static std::string
Key(int i
) {
206 snprintf(buf
, sizeof(buf
), "key_______%06d", i
);
207 return std::string(buf
);
209 static std::string
Uint64Key(uint64_t i
) {
212 memcpy(&str
[0], static_cast<void*>(&i
), 8);
217 TEST_F(CuckooTableDBTest
, Uint64Comparator
) {
218 Options options
= CurrentOptions();
219 options
.comparator
= test::Uint64Comparator();
220 DestroyAndReopen(&options
);
222 ASSERT_OK(Put(Uint64Key(1), "v1"));
223 ASSERT_OK(Put(Uint64Key(2), "v2"));
224 ASSERT_OK(Put(Uint64Key(3), "v3"));
225 ASSERT_OK(dbfull()->TEST_FlushMemTable());
227 ASSERT_EQ("v1", Get(Uint64Key(1)));
228 ASSERT_EQ("v2", Get(Uint64Key(2)));
229 ASSERT_EQ("v3", Get(Uint64Key(3)));
230 ASSERT_EQ("NOT_FOUND", Get(Uint64Key(4)));
233 ASSERT_OK(Delete(Uint64Key(2))); // Delete.
234 ASSERT_OK(dbfull()->TEST_FlushMemTable());
235 ASSERT_OK(Put(Uint64Key(3), "v0")); // Update.
236 ASSERT_OK(Put(Uint64Key(4), "v4"));
237 ASSERT_OK(dbfull()->TEST_FlushMemTable());
238 ASSERT_EQ("v1", Get(Uint64Key(1)));
239 ASSERT_EQ("NOT_FOUND", Get(Uint64Key(2)));
240 ASSERT_EQ("v0", Get(Uint64Key(3)));
241 ASSERT_EQ("v4", Get(Uint64Key(4)));
244 TEST_F(CuckooTableDBTest
, CompactionIntoMultipleFiles
) {
245 // Create a big L0 file and check it compacts into multiple files in L1.
246 Options options
= CurrentOptions();
247 options
.write_buffer_size
= 270 << 10;
248 // Two SST files should be created, each containing 14 keys.
249 // Number of buckets will be 16. Total size ~156 KB.
250 options
.target_file_size_base
= 160 << 10;
253 // Write 28 values, each 10016 B ~ 10KB
254 for (int idx
= 0; idx
< 28; ++idx
) {
255 ASSERT_OK(Put(Key(idx
), std::string(10000, 'a' + char(idx
))));
257 ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
258 ASSERT_EQ("1", FilesPerLevel());
260 ASSERT_OK(dbfull()->TEST_CompactRange(0, nullptr, nullptr, nullptr,
261 true /* disallow trivial move */));
262 ASSERT_EQ("0,2", FilesPerLevel());
263 for (int idx
= 0; idx
< 28; ++idx
) {
264 ASSERT_EQ(std::string(10000, 'a' + char(idx
)), Get(Key(idx
)));
268 TEST_F(CuckooTableDBTest
, SameKeyInsertedInTwoDifferentFilesAndCompacted
) {
269 // Insert same key twice so that they go to different SST files. Then wait for
270 // compaction and check if the latest value is stored and old value removed.
271 Options options
= CurrentOptions();
272 options
.write_buffer_size
= 100 << 10; // 100KB
273 options
.level0_file_num_compaction_trigger
= 2;
276 // Write 11 values, each 10016 B
277 for (int idx
= 0; idx
< 11; ++idx
) {
278 ASSERT_OK(Put(Key(idx
), std::string(10000, 'a')));
280 ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
281 ASSERT_EQ("1", FilesPerLevel());
283 // Generate one more file in level-0, and should trigger level-0 compaction
284 for (int idx
= 0; idx
< 11; ++idx
) {
285 ASSERT_OK(Put(Key(idx
), std::string(10000, 'a' + char(idx
))));
287 ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
288 ASSERT_OK(dbfull()->TEST_CompactRange(0, nullptr, nullptr));
290 ASSERT_EQ("0,1", FilesPerLevel());
291 for (int idx
= 0; idx
< 11; ++idx
) {
292 ASSERT_EQ(std::string(10000, 'a' + char(idx
)), Get(Key(idx
)));
296 TEST_F(CuckooTableDBTest
, AdaptiveTable
) {
297 Options options
= CurrentOptions();
299 // Ensure options compatible with PlainTable
300 options
.prefix_extractor
.reset(NewCappedPrefixTransform(8));
302 // Write some keys using cuckoo table.
303 options
.table_factory
.reset(NewCuckooTableFactory());
306 ASSERT_OK(Put("key1", "v1"));
307 ASSERT_OK(Put("key2", "v2"));
308 ASSERT_OK(Put("key3", "v3"));
309 ASSERT_OK(dbfull()->TEST_FlushMemTable());
311 // Write some keys using plain table.
312 std::shared_ptr
<TableFactory
> block_based_factory(
313 NewBlockBasedTableFactory());
314 std::shared_ptr
<TableFactory
> plain_table_factory(NewPlainTableFactory());
315 std::shared_ptr
<TableFactory
> cuckoo_table_factory(NewCuckooTableFactory());
316 options
.create_if_missing
= false;
317 options
.table_factory
.reset(
318 NewAdaptiveTableFactory(plain_table_factory
, block_based_factory
,
319 plain_table_factory
, cuckoo_table_factory
));
321 ASSERT_OK(Put("key4", "v4"));
322 ASSERT_OK(Put("key1", "v5"));
323 ASSERT_OK(dbfull()->TEST_FlushMemTable());
325 // Write some keys using block based table.
326 options
.table_factory
.reset(
327 NewAdaptiveTableFactory(block_based_factory
, block_based_factory
,
328 plain_table_factory
, cuckoo_table_factory
));
330 ASSERT_OK(Put("key5", "v6"));
331 ASSERT_OK(Put("key2", "v7"));
332 ASSERT_OK(dbfull()->TEST_FlushMemTable());
334 ASSERT_EQ("v5", Get("key1"));
335 ASSERT_EQ("v7", Get("key2"));
336 ASSERT_EQ("v3", Get("key3"));
337 ASSERT_EQ("v4", Get("key4"));
338 ASSERT_EQ("v6", Get("key5"));
340 } // namespace ROCKSDB_NAMESPACE
342 int main(int argc
, char** argv
) {
343 if (ROCKSDB_NAMESPACE::port::kLittleEndian
) {
344 ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
345 ::testing::InitGoogleTest(&argc
, argv
);
346 return RUN_ALL_TESTS();
348 fprintf(stderr
, "SKIPPED as Cuckoo table doesn't support Big Endian\n");
356 int main(int /*argc*/, char** /*argv*/) {
357 fprintf(stderr
, "SKIPPED as Cuckoo table is not supported in ROCKSDB_LITE\n");
361 #endif // ROCKSDB_LITE