1 // Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
2 // This source code is licensed under the BSD-style license found in the
3 // LICENSE file in the root directory of this source tree. An additional grant
4 // of patent rights can be found in the PATENTS file in the same directory.
8 #include "db/db_impl.h"
9 #include "rocksdb/db.h"
10 #include "rocksdb/env.h"
11 #include "table/cuckoo_table_factory.h"
12 #include "table/cuckoo_table_reader.h"
13 #include "table/meta_blocks.h"
14 #include "util/string_util.h"
15 #include "util/testharness.h"
16 #include "util/testutil.h"
20 class CuckooTableDBTest
: public testing::Test
{
27 CuckooTableDBTest() : env_(Env::Default()) {
28 dbname_
= test::TmpDir() + "/cuckoo_table_db_test";
29 EXPECT_OK(DestroyDB(dbname_
, Options()));
34 ~CuckooTableDBTest() {
36 EXPECT_OK(DestroyDB(dbname_
, Options()));
39 Options
CurrentOptions() {
41 options
.table_factory
.reset(NewCuckooTableFactory());
42 options
.memtable_factory
.reset(NewHashLinkListRepFactory(4, 0, 3, true));
43 options
.allow_mmap_reads
= true;
44 options
.create_if_missing
= true;
45 options
.allow_concurrent_memtable_write
= false;
50 return reinterpret_cast<DBImpl
*>(db_
);
53 // The following util methods are copied from plain_table_db_test.
54 void Reopen(Options
* options
= nullptr) {
58 if (options
!= nullptr) {
61 opts
= CurrentOptions();
62 opts
.create_if_missing
= true;
64 ASSERT_OK(DB::Open(opts
, dbname_
, &db_
));
67 Status
Put(const Slice
& k
, const Slice
& v
) {
68 return db_
->Put(WriteOptions(), k
, v
);
71 Status
Delete(const std::string
& k
) {
72 return db_
->Delete(WriteOptions(), k
);
75 std::string
Get(const std::string
& k
) {
78 Status s
= db_
->Get(options
, k
, &result
);
82 result
= s
.ToString();
87 int NumTableFilesAtLevel(int level
) {
89 EXPECT_TRUE(db_
->GetProperty(
90 "rocksdb.num-files-at-level" + NumberToString(level
), &property
));
91 return atoi(property
.c_str());
94 // Return spread of files per level
95 std::string
FilesPerLevel() {
97 size_t last_non_zero_offset
= 0;
98 for (int level
= 0; level
< db_
->NumberLevels(); level
++) {
99 int f
= NumTableFilesAtLevel(level
);
101 snprintf(buf
, sizeof(buf
), "%s%d", (level
? "," : ""), f
);
104 last_non_zero_offset
= result
.size();
107 result
.resize(last_non_zero_offset
);
112 TEST_F(CuckooTableDBTest
, Flush
) {
113 // Try with empty DB first.
114 ASSERT_TRUE(dbfull() != nullptr);
115 ASSERT_EQ("NOT_FOUND", Get("key2"));
117 // Add some values to db.
118 Options options
= CurrentOptions();
121 ASSERT_OK(Put("key1", "v1"));
122 ASSERT_OK(Put("key2", "v2"));
123 ASSERT_OK(Put("key3", "v3"));
124 dbfull()->TEST_FlushMemTable();
126 TablePropertiesCollection ptc
;
127 reinterpret_cast<DB
*>(dbfull())->GetPropertiesOfAllTables(&ptc
);
128 ASSERT_EQ(1U, ptc
.size());
129 ASSERT_EQ(3U, ptc
.begin()->second
->num_entries
);
130 ASSERT_EQ("1", FilesPerLevel());
132 ASSERT_EQ("v1", Get("key1"));
133 ASSERT_EQ("v2", Get("key2"));
134 ASSERT_EQ("v3", Get("key3"));
135 ASSERT_EQ("NOT_FOUND", Get("key4"));
137 // Now add more keys and flush.
138 ASSERT_OK(Put("key4", "v4"));
139 ASSERT_OK(Put("key5", "v5"));
140 ASSERT_OK(Put("key6", "v6"));
141 dbfull()->TEST_FlushMemTable();
143 reinterpret_cast<DB
*>(dbfull())->GetPropertiesOfAllTables(&ptc
);
144 ASSERT_EQ(2U, ptc
.size());
145 auto row
= ptc
.begin();
146 ASSERT_EQ(3U, row
->second
->num_entries
);
147 ASSERT_EQ(3U, (++row
)->second
->num_entries
);
148 ASSERT_EQ("2", FilesPerLevel());
149 ASSERT_EQ("v1", Get("key1"));
150 ASSERT_EQ("v2", Get("key2"));
151 ASSERT_EQ("v3", Get("key3"));
152 ASSERT_EQ("v4", Get("key4"));
153 ASSERT_EQ("v5", Get("key5"));
154 ASSERT_EQ("v6", Get("key6"));
156 ASSERT_OK(Delete("key6"));
157 ASSERT_OK(Delete("key5"));
158 ASSERT_OK(Delete("key4"));
159 dbfull()->TEST_FlushMemTable();
160 reinterpret_cast<DB
*>(dbfull())->GetPropertiesOfAllTables(&ptc
);
161 ASSERT_EQ(3U, ptc
.size());
163 ASSERT_EQ(3U, row
->second
->num_entries
);
164 ASSERT_EQ(3U, (++row
)->second
->num_entries
);
165 ASSERT_EQ(3U, (++row
)->second
->num_entries
);
166 ASSERT_EQ("3", FilesPerLevel());
167 ASSERT_EQ("v1", Get("key1"));
168 ASSERT_EQ("v2", Get("key2"));
169 ASSERT_EQ("v3", Get("key3"));
170 ASSERT_EQ("NOT_FOUND", Get("key4"));
171 ASSERT_EQ("NOT_FOUND", Get("key5"));
172 ASSERT_EQ("NOT_FOUND", Get("key6"));
175 TEST_F(CuckooTableDBTest
, FlushWithDuplicateKeys
) {
176 Options options
= CurrentOptions();
178 ASSERT_OK(Put("key1", "v1"));
179 ASSERT_OK(Put("key2", "v2"));
180 ASSERT_OK(Put("key1", "v3")); // Duplicate
181 dbfull()->TEST_FlushMemTable();
183 TablePropertiesCollection ptc
;
184 reinterpret_cast<DB
*>(dbfull())->GetPropertiesOfAllTables(&ptc
);
185 ASSERT_EQ(1U, ptc
.size());
186 ASSERT_EQ(2U, ptc
.begin()->second
->num_entries
);
187 ASSERT_EQ("1", FilesPerLevel());
188 ASSERT_EQ("v3", Get("key1"));
189 ASSERT_EQ("v2", Get("key2"));
193 static std::string
Key(int i
) {
195 snprintf(buf
, sizeof(buf
), "key_______%06d", i
);
196 return std::string(buf
);
198 static std::string
Uint64Key(uint64_t i
) {
201 memcpy(&str
[0], static_cast<void*>(&i
), 8);
206 TEST_F(CuckooTableDBTest
, Uint64Comparator
) {
207 Options options
= CurrentOptions();
208 options
.comparator
= test::Uint64Comparator();
211 ASSERT_OK(Put(Uint64Key(1), "v1"));
212 ASSERT_OK(Put(Uint64Key(2), "v2"));
213 ASSERT_OK(Put(Uint64Key(3), "v3"));
214 dbfull()->TEST_FlushMemTable();
216 ASSERT_EQ("v1", Get(Uint64Key(1)));
217 ASSERT_EQ("v2", Get(Uint64Key(2)));
218 ASSERT_EQ("v3", Get(Uint64Key(3)));
219 ASSERT_EQ("NOT_FOUND", Get(Uint64Key(4)));
222 ASSERT_OK(Delete(Uint64Key(2))); // Delete.
223 dbfull()->TEST_FlushMemTable();
224 ASSERT_OK(Put(Uint64Key(3), "v0")); // Update.
225 ASSERT_OK(Put(Uint64Key(4), "v4"));
226 dbfull()->TEST_FlushMemTable();
227 ASSERT_EQ("v1", Get(Uint64Key(1)));
228 ASSERT_EQ("NOT_FOUND", Get(Uint64Key(2)));
229 ASSERT_EQ("v0", Get(Uint64Key(3)));
230 ASSERT_EQ("v4", Get(Uint64Key(4)));
233 TEST_F(CuckooTableDBTest
, CompactionIntoMultipleFiles
) {
234 // Create a big L0 file and check it compacts into multiple files in L1.
235 Options options
= CurrentOptions();
236 options
.write_buffer_size
= 270 << 10;
237 // Two SST files should be created, each containing 14 keys.
238 // Number of buckets will be 16. Total size ~156 KB.
239 options
.target_file_size_base
= 160 << 10;
242 // Write 28 values, each 10016 B ~ 10KB
243 for (int idx
= 0; idx
< 28; ++idx
) {
244 ASSERT_OK(Put(Key(idx
), std::string(10000, 'a' + idx
)));
246 dbfull()->TEST_WaitForFlushMemTable();
247 ASSERT_EQ("1", FilesPerLevel());
249 dbfull()->TEST_CompactRange(0, nullptr, nullptr, nullptr,
250 true /* disallow trivial move */);
251 ASSERT_EQ("0,2", FilesPerLevel());
252 for (int idx
= 0; idx
< 28; ++idx
) {
253 ASSERT_EQ(std::string(10000, 'a' + idx
), Get(Key(idx
)));
257 TEST_F(CuckooTableDBTest
, SameKeyInsertedInTwoDifferentFilesAndCompacted
) {
258 // Insert same key twice so that they go to different SST files. Then wait for
259 // compaction and check if the latest value is stored and old value removed.
260 Options options
= CurrentOptions();
261 options
.write_buffer_size
= 100 << 10; // 100KB
262 options
.level0_file_num_compaction_trigger
= 2;
265 // Write 11 values, each 10016 B
266 for (int idx
= 0; idx
< 11; ++idx
) {
267 ASSERT_OK(Put(Key(idx
), std::string(10000, 'a')));
269 dbfull()->TEST_WaitForFlushMemTable();
270 ASSERT_EQ("1", FilesPerLevel());
272 // Generate one more file in level-0, and should trigger level-0 compaction
273 for (int idx
= 0; idx
< 11; ++idx
) {
274 ASSERT_OK(Put(Key(idx
), std::string(10000, 'a' + idx
)));
276 dbfull()->TEST_WaitForFlushMemTable();
277 dbfull()->TEST_CompactRange(0, nullptr, nullptr);
279 ASSERT_EQ("0,1", FilesPerLevel());
280 for (int idx
= 0; idx
< 11; ++idx
) {
281 ASSERT_EQ(std::string(10000, 'a' + idx
), Get(Key(idx
)));
285 TEST_F(CuckooTableDBTest
, AdaptiveTable
) {
286 Options options
= CurrentOptions();
288 // Write some keys using cuckoo table.
289 options
.table_factory
.reset(NewCuckooTableFactory());
292 ASSERT_OK(Put("key1", "v1"));
293 ASSERT_OK(Put("key2", "v2"));
294 ASSERT_OK(Put("key3", "v3"));
295 dbfull()->TEST_FlushMemTable();
297 // Write some keys using plain table.
298 options
.create_if_missing
= false;
299 options
.table_factory
.reset(NewPlainTableFactory());
301 ASSERT_OK(Put("key4", "v4"));
302 ASSERT_OK(Put("key1", "v5"));
303 dbfull()->TEST_FlushMemTable();
305 // Write some keys using block based table.
306 std::shared_ptr
<TableFactory
> block_based_factory(
307 NewBlockBasedTableFactory());
308 options
.table_factory
.reset(NewAdaptiveTableFactory(block_based_factory
));
310 ASSERT_OK(Put("key5", "v6"));
311 ASSERT_OK(Put("key2", "v7"));
312 dbfull()->TEST_FlushMemTable();
314 ASSERT_EQ("v5", Get("key1"));
315 ASSERT_EQ("v7", Get("key2"));
316 ASSERT_EQ("v3", Get("key3"));
317 ASSERT_EQ("v4", Get("key4"));
318 ASSERT_EQ("v6", Get("key5"));
320 } // namespace rocksdb
322 int main(int argc
, char** argv
) {
323 if (rocksdb::port::kLittleEndian
) {
324 ::testing::InitGoogleTest(&argc
, argv
);
325 return RUN_ALL_TESTS();
328 fprintf(stderr
, "SKIPPED as Cuckoo table doesn't support Big Endian\n");
336 int main(int argc
, char** argv
) {
337 fprintf(stderr
, "SKIPPED as Cuckoo table is not supported in ROCKSDB_LITE\n");
341 #endif // ROCKSDB_LITE