]>
Commit | Line | Data |
---|---|---|
7c673cae | 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. |
11fdf7f2 TL |
2 | // This source code is licensed under both the GPLv2 (found in the |
3 | // COPYING file in the root directory) and Apache 2.0 License | |
4 | // (found in the LICENSE.Apache file in the root directory). | |
7c673cae FG |
5 | |
6 | #ifndef ROCKSDB_LITE | |
7 | ||
8 | #include "db/db_impl.h" | |
9 | #include "rocksdb/db.h" | |
10 | #include "rocksdb/env.h" | |
11 | #include "table/cuckoo_table_factory.h" | |
12 | #include "table/cuckoo_table_reader.h" | |
13 | #include "table/meta_blocks.h" | |
14 | #include "util/string_util.h" | |
15 | #include "util/testharness.h" | |
16 | #include "util/testutil.h" | |
17 | ||
18 | namespace rocksdb { | |
19 | ||
20 | class CuckooTableDBTest : public testing::Test { | |
21 | private: | |
22 | std::string dbname_; | |
23 | Env* env_; | |
24 | DB* db_; | |
25 | ||
26 | public: | |
27 | CuckooTableDBTest() : env_(Env::Default()) { | |
11fdf7f2 | 28 | dbname_ = test::PerThreadDBPath("cuckoo_table_db_test"); |
7c673cae FG |
29 | EXPECT_OK(DestroyDB(dbname_, Options())); |
30 | db_ = nullptr; | |
31 | Reopen(); | |
32 | } | |
33 | ||
494da23a | 34 | ~CuckooTableDBTest() override { |
7c673cae FG |
35 | delete db_; |
36 | EXPECT_OK(DestroyDB(dbname_, Options())); | |
37 | } | |
38 | ||
39 | Options CurrentOptions() { | |
40 | Options options; | |
41 | options.table_factory.reset(NewCuckooTableFactory()); | |
42 | options.memtable_factory.reset(NewHashLinkListRepFactory(4, 0, 3, true)); | |
43 | options.allow_mmap_reads = true; | |
44 | options.create_if_missing = true; | |
45 | options.allow_concurrent_memtable_write = false; | |
46 | return options; | |
47 | } | |
48 | ||
49 | DBImpl* dbfull() { | |
50 | return reinterpret_cast<DBImpl*>(db_); | |
51 | } | |
52 | ||
53 | // The following util methods are copied from plain_table_db_test. | |
54 | void Reopen(Options* options = nullptr) { | |
55 | delete db_; | |
56 | db_ = nullptr; | |
57 | Options opts; | |
58 | if (options != nullptr) { | |
59 | opts = *options; | |
60 | } else { | |
61 | opts = CurrentOptions(); | |
62 | opts.create_if_missing = true; | |
63 | } | |
64 | ASSERT_OK(DB::Open(opts, dbname_, &db_)); | |
65 | } | |
66 | ||
67 | Status Put(const Slice& k, const Slice& v) { | |
68 | return db_->Put(WriteOptions(), k, v); | |
69 | } | |
70 | ||
71 | Status Delete(const std::string& k) { | |
72 | return db_->Delete(WriteOptions(), k); | |
73 | } | |
74 | ||
75 | std::string Get(const std::string& k) { | |
76 | ReadOptions options; | |
77 | std::string result; | |
78 | Status s = db_->Get(options, k, &result); | |
79 | if (s.IsNotFound()) { | |
80 | result = "NOT_FOUND"; | |
81 | } else if (!s.ok()) { | |
82 | result = s.ToString(); | |
83 | } | |
84 | return result; | |
85 | } | |
86 | ||
87 | int NumTableFilesAtLevel(int level) { | |
88 | std::string property; | |
89 | EXPECT_TRUE(db_->GetProperty( | |
90 | "rocksdb.num-files-at-level" + NumberToString(level), &property)); | |
91 | return atoi(property.c_str()); | |
92 | } | |
93 | ||
94 | // Return spread of files per level | |
95 | std::string FilesPerLevel() { | |
96 | std::string result; | |
97 | size_t last_non_zero_offset = 0; | |
98 | for (int level = 0; level < db_->NumberLevels(); level++) { | |
99 | int f = NumTableFilesAtLevel(level); | |
100 | char buf[100]; | |
101 | snprintf(buf, sizeof(buf), "%s%d", (level ? "," : ""), f); | |
102 | result += buf; | |
103 | if (f > 0) { | |
104 | last_non_zero_offset = result.size(); | |
105 | } | |
106 | } | |
107 | result.resize(last_non_zero_offset); | |
108 | return result; | |
109 | } | |
110 | }; | |
111 | ||
112 | TEST_F(CuckooTableDBTest, Flush) { | |
113 | // Try with empty DB first. | |
114 | ASSERT_TRUE(dbfull() != nullptr); | |
115 | ASSERT_EQ("NOT_FOUND", Get("key2")); | |
116 | ||
117 | // Add some values to db. | |
118 | Options options = CurrentOptions(); | |
119 | Reopen(&options); | |
120 | ||
121 | ASSERT_OK(Put("key1", "v1")); | |
122 | ASSERT_OK(Put("key2", "v2")); | |
123 | ASSERT_OK(Put("key3", "v3")); | |
124 | dbfull()->TEST_FlushMemTable(); | |
125 | ||
126 | TablePropertiesCollection ptc; | |
127 | reinterpret_cast<DB*>(dbfull())->GetPropertiesOfAllTables(&ptc); | |
128 | ASSERT_EQ(1U, ptc.size()); | |
129 | ASSERT_EQ(3U, ptc.begin()->second->num_entries); | |
130 | ASSERT_EQ("1", FilesPerLevel()); | |
131 | ||
132 | ASSERT_EQ("v1", Get("key1")); | |
133 | ASSERT_EQ("v2", Get("key2")); | |
134 | ASSERT_EQ("v3", Get("key3")); | |
135 | ASSERT_EQ("NOT_FOUND", Get("key4")); | |
136 | ||
137 | // Now add more keys and flush. | |
138 | ASSERT_OK(Put("key4", "v4")); | |
139 | ASSERT_OK(Put("key5", "v5")); | |
140 | ASSERT_OK(Put("key6", "v6")); | |
141 | dbfull()->TEST_FlushMemTable(); | |
142 | ||
143 | reinterpret_cast<DB*>(dbfull())->GetPropertiesOfAllTables(&ptc); | |
144 | ASSERT_EQ(2U, ptc.size()); | |
145 | auto row = ptc.begin(); | |
146 | ASSERT_EQ(3U, row->second->num_entries); | |
147 | ASSERT_EQ(3U, (++row)->second->num_entries); | |
148 | ASSERT_EQ("2", FilesPerLevel()); | |
149 | ASSERT_EQ("v1", Get("key1")); | |
150 | ASSERT_EQ("v2", Get("key2")); | |
151 | ASSERT_EQ("v3", Get("key3")); | |
152 | ASSERT_EQ("v4", Get("key4")); | |
153 | ASSERT_EQ("v5", Get("key5")); | |
154 | ASSERT_EQ("v6", Get("key6")); | |
155 | ||
156 | ASSERT_OK(Delete("key6")); | |
157 | ASSERT_OK(Delete("key5")); | |
158 | ASSERT_OK(Delete("key4")); | |
159 | dbfull()->TEST_FlushMemTable(); | |
160 | reinterpret_cast<DB*>(dbfull())->GetPropertiesOfAllTables(&ptc); | |
161 | ASSERT_EQ(3U, ptc.size()); | |
162 | row = ptc.begin(); | |
163 | ASSERT_EQ(3U, row->second->num_entries); | |
164 | ASSERT_EQ(3U, (++row)->second->num_entries); | |
165 | ASSERT_EQ(3U, (++row)->second->num_entries); | |
166 | ASSERT_EQ("3", FilesPerLevel()); | |
167 | ASSERT_EQ("v1", Get("key1")); | |
168 | ASSERT_EQ("v2", Get("key2")); | |
169 | ASSERT_EQ("v3", Get("key3")); | |
170 | ASSERT_EQ("NOT_FOUND", Get("key4")); | |
171 | ASSERT_EQ("NOT_FOUND", Get("key5")); | |
172 | ASSERT_EQ("NOT_FOUND", Get("key6")); | |
173 | } | |
174 | ||
175 | TEST_F(CuckooTableDBTest, FlushWithDuplicateKeys) { | |
176 | Options options = CurrentOptions(); | |
177 | Reopen(&options); | |
178 | ASSERT_OK(Put("key1", "v1")); | |
179 | ASSERT_OK(Put("key2", "v2")); | |
180 | ASSERT_OK(Put("key1", "v3")); // Duplicate | |
181 | dbfull()->TEST_FlushMemTable(); | |
182 | ||
183 | TablePropertiesCollection ptc; | |
184 | reinterpret_cast<DB*>(dbfull())->GetPropertiesOfAllTables(&ptc); | |
185 | ASSERT_EQ(1U, ptc.size()); | |
186 | ASSERT_EQ(2U, ptc.begin()->second->num_entries); | |
187 | ASSERT_EQ("1", FilesPerLevel()); | |
188 | ASSERT_EQ("v3", Get("key1")); | |
189 | ASSERT_EQ("v2", Get("key2")); | |
190 | } | |
191 | ||
192 | namespace { | |
193 | static std::string Key(int i) { | |
194 | char buf[100]; | |
195 | snprintf(buf, sizeof(buf), "key_______%06d", i); | |
196 | return std::string(buf); | |
197 | } | |
198 | static std::string Uint64Key(uint64_t i) { | |
199 | std::string str; | |
200 | str.resize(8); | |
201 | memcpy(&str[0], static_cast<void*>(&i), 8); | |
202 | return str; | |
203 | } | |
204 | } // namespace. | |
205 | ||
206 | TEST_F(CuckooTableDBTest, Uint64Comparator) { | |
207 | Options options = CurrentOptions(); | |
208 | options.comparator = test::Uint64Comparator(); | |
209 | Reopen(&options); | |
210 | ||
211 | ASSERT_OK(Put(Uint64Key(1), "v1")); | |
212 | ASSERT_OK(Put(Uint64Key(2), "v2")); | |
213 | ASSERT_OK(Put(Uint64Key(3), "v3")); | |
214 | dbfull()->TEST_FlushMemTable(); | |
215 | ||
216 | ASSERT_EQ("v1", Get(Uint64Key(1))); | |
217 | ASSERT_EQ("v2", Get(Uint64Key(2))); | |
218 | ASSERT_EQ("v3", Get(Uint64Key(3))); | |
219 | ASSERT_EQ("NOT_FOUND", Get(Uint64Key(4))); | |
220 | ||
221 | // Add more keys. | |
222 | ASSERT_OK(Delete(Uint64Key(2))); // Delete. | |
223 | dbfull()->TEST_FlushMemTable(); | |
224 | ASSERT_OK(Put(Uint64Key(3), "v0")); // Update. | |
225 | ASSERT_OK(Put(Uint64Key(4), "v4")); | |
226 | dbfull()->TEST_FlushMemTable(); | |
227 | ASSERT_EQ("v1", Get(Uint64Key(1))); | |
228 | ASSERT_EQ("NOT_FOUND", Get(Uint64Key(2))); | |
229 | ASSERT_EQ("v0", Get(Uint64Key(3))); | |
230 | ASSERT_EQ("v4", Get(Uint64Key(4))); | |
231 | } | |
232 | ||
233 | TEST_F(CuckooTableDBTest, CompactionIntoMultipleFiles) { | |
234 | // Create a big L0 file and check it compacts into multiple files in L1. | |
235 | Options options = CurrentOptions(); | |
236 | options.write_buffer_size = 270 << 10; | |
237 | // Two SST files should be created, each containing 14 keys. | |
238 | // Number of buckets will be 16. Total size ~156 KB. | |
239 | options.target_file_size_base = 160 << 10; | |
240 | Reopen(&options); | |
241 | ||
242 | // Write 28 values, each 10016 B ~ 10KB | |
243 | for (int idx = 0; idx < 28; ++idx) { | |
11fdf7f2 | 244 | ASSERT_OK(Put(Key(idx), std::string(10000, 'a' + char(idx)))); |
7c673cae FG |
245 | } |
246 | dbfull()->TEST_WaitForFlushMemTable(); | |
247 | ASSERT_EQ("1", FilesPerLevel()); | |
248 | ||
249 | dbfull()->TEST_CompactRange(0, nullptr, nullptr, nullptr, | |
250 | true /* disallow trivial move */); | |
251 | ASSERT_EQ("0,2", FilesPerLevel()); | |
252 | for (int idx = 0; idx < 28; ++idx) { | |
11fdf7f2 | 253 | ASSERT_EQ(std::string(10000, 'a' + char(idx)), Get(Key(idx))); |
7c673cae FG |
254 | } |
255 | } | |
256 | ||
257 | TEST_F(CuckooTableDBTest, SameKeyInsertedInTwoDifferentFilesAndCompacted) { | |
258 | // Insert same key twice so that they go to different SST files. Then wait for | |
259 | // compaction and check if the latest value is stored and old value removed. | |
260 | Options options = CurrentOptions(); | |
261 | options.write_buffer_size = 100 << 10; // 100KB | |
262 | options.level0_file_num_compaction_trigger = 2; | |
263 | Reopen(&options); | |
264 | ||
265 | // Write 11 values, each 10016 B | |
266 | for (int idx = 0; idx < 11; ++idx) { | |
267 | ASSERT_OK(Put(Key(idx), std::string(10000, 'a'))); | |
268 | } | |
269 | dbfull()->TEST_WaitForFlushMemTable(); | |
270 | ASSERT_EQ("1", FilesPerLevel()); | |
271 | ||
272 | // Generate one more file in level-0, and should trigger level-0 compaction | |
273 | for (int idx = 0; idx < 11; ++idx) { | |
11fdf7f2 | 274 | ASSERT_OK(Put(Key(idx), std::string(10000, 'a' + char(idx)))); |
7c673cae FG |
275 | } |
276 | dbfull()->TEST_WaitForFlushMemTable(); | |
277 | dbfull()->TEST_CompactRange(0, nullptr, nullptr); | |
278 | ||
279 | ASSERT_EQ("0,1", FilesPerLevel()); | |
280 | for (int idx = 0; idx < 11; ++idx) { | |
11fdf7f2 | 281 | ASSERT_EQ(std::string(10000, 'a' + char(idx)), Get(Key(idx))); |
7c673cae FG |
282 | } |
283 | } | |
284 | ||
285 | TEST_F(CuckooTableDBTest, AdaptiveTable) { | |
286 | Options options = CurrentOptions(); | |
287 | ||
288 | // Write some keys using cuckoo table. | |
289 | options.table_factory.reset(NewCuckooTableFactory()); | |
290 | Reopen(&options); | |
291 | ||
292 | ASSERT_OK(Put("key1", "v1")); | |
293 | ASSERT_OK(Put("key2", "v2")); | |
294 | ASSERT_OK(Put("key3", "v3")); | |
295 | dbfull()->TEST_FlushMemTable(); | |
296 | ||
297 | // Write some keys using plain table. | |
298 | options.create_if_missing = false; | |
299 | options.table_factory.reset(NewPlainTableFactory()); | |
300 | Reopen(&options); | |
301 | ASSERT_OK(Put("key4", "v4")); | |
302 | ASSERT_OK(Put("key1", "v5")); | |
303 | dbfull()->TEST_FlushMemTable(); | |
304 | ||
305 | // Write some keys using block based table. | |
306 | std::shared_ptr<TableFactory> block_based_factory( | |
307 | NewBlockBasedTableFactory()); | |
308 | options.table_factory.reset(NewAdaptiveTableFactory(block_based_factory)); | |
309 | Reopen(&options); | |
310 | ASSERT_OK(Put("key5", "v6")); | |
311 | ASSERT_OK(Put("key2", "v7")); | |
312 | dbfull()->TEST_FlushMemTable(); | |
313 | ||
314 | ASSERT_EQ("v5", Get("key1")); | |
315 | ASSERT_EQ("v7", Get("key2")); | |
316 | ASSERT_EQ("v3", Get("key3")); | |
317 | ASSERT_EQ("v4", Get("key4")); | |
318 | ASSERT_EQ("v6", Get("key5")); | |
319 | } | |
320 | } // namespace rocksdb | |
321 | ||
322 | int main(int argc, char** argv) { | |
323 | if (rocksdb::port::kLittleEndian) { | |
324 | ::testing::InitGoogleTest(&argc, argv); | |
325 | return RUN_ALL_TESTS(); | |
326 | } | |
327 | else { | |
328 | fprintf(stderr, "SKIPPED as Cuckoo table doesn't support Big Endian\n"); | |
329 | return 0; | |
330 | } | |
331 | } | |
332 | ||
333 | #else | |
334 | #include <stdio.h> | |
335 | ||
11fdf7f2 | 336 | int main(int /*argc*/, char** /*argv*/) { |
7c673cae FG |
337 | fprintf(stderr, "SKIPPED as Cuckoo table is not supported in ROCKSDB_LITE\n"); |
338 | return 0; | |
339 | } | |
340 | ||
341 | #endif // ROCKSDB_LITE |