]>
Commit | Line | Data |
---|---|---|
7c673cae | 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. |
11fdf7f2 TL |
2 | // This source code is licensed under both the GPLv2 (found in the |
3 | // COPYING file in the root directory) and Apache 2.0 License | |
4 | // (found in the LICENSE.Apache file in the root directory). | |
7c673cae FG |
5 | // |
6 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. | |
7 | // Use of this source code is governed by a BSD-style license that can be | |
8 | // found in the LICENSE file. See the AUTHORS file for names of contributors. | |
9 | ||
10 | #include <unordered_set> | |
11 | #include <vector> | |
12 | ||
13 | #include "db/db_test_util.h" | |
14 | #include "port/stack_trace.h" | |
15 | #include "rocksdb/db.h" | |
11fdf7f2 | 16 | #include "rocksdb/utilities/table_properties_collectors.h" |
f67539c2 TL |
17 | #include "test_util/testharness.h" |
18 | #include "test_util/testutil.h" | |
7c673cae FG |
19 | |
20 | #ifndef ROCKSDB_LITE | |
21 | ||
f67539c2 | 22 | namespace ROCKSDB_NAMESPACE { |
7c673cae FG |
23 | |
24 | // A helper function that ensures the table properties returned in | |
25 | // `GetPropertiesOfAllTablesTest` is correct. | |
26 | // This test assumes entries size is different for each of the tables. | |
27 | namespace { | |
28 | ||
29 | void VerifyTableProperties(DB* db, uint64_t expected_entries_size) { | |
30 | TablePropertiesCollection props; | |
31 | ASSERT_OK(db->GetPropertiesOfAllTables(&props)); | |
32 | ||
33 | ASSERT_EQ(4U, props.size()); | |
34 | std::unordered_set<uint64_t> unique_entries; | |
35 | ||
36 | // Indirect test | |
37 | uint64_t sum = 0; | |
38 | for (const auto& item : props) { | |
39 | unique_entries.insert(item.second->num_entries); | |
40 | sum += item.second->num_entries; | |
41 | } | |
42 | ||
43 | ASSERT_EQ(props.size(), unique_entries.size()); | |
44 | ASSERT_EQ(expected_entries_size, sum); | |
45 | } | |
46 | } // namespace | |
47 | ||
48 | class DBTablePropertiesTest : public DBTestBase { | |
49 | public: | |
50 | DBTablePropertiesTest() : DBTestBase("/db_table_properties_test") {} | |
51 | TablePropertiesCollection TestGetPropertiesOfTablesInRange( | |
52 | std::vector<Range> ranges, std::size_t* num_properties = nullptr, | |
53 | std::size_t* num_files = nullptr); | |
54 | }; | |
55 | ||
56 | TEST_F(DBTablePropertiesTest, GetPropertiesOfAllTablesTest) { | |
57 | Options options = CurrentOptions(); | |
58 | options.level0_file_num_compaction_trigger = 8; | |
59 | Reopen(options); | |
60 | // Create 4 tables | |
61 | for (int table = 0; table < 4; ++table) { | |
62 | for (int i = 0; i < 10 + table; ++i) { | |
63 | db_->Put(WriteOptions(), ToString(table * 100 + i), "val"); | |
64 | } | |
65 | db_->Flush(FlushOptions()); | |
66 | } | |
67 | ||
68 | // 1. Read table properties directly from file | |
69 | Reopen(options); | |
70 | VerifyTableProperties(db_, 10 + 11 + 12 + 13); | |
71 | ||
72 | // 2. Put two tables to table cache and | |
73 | Reopen(options); | |
74 | // fetch key from 1st and 2nd table, which will internally place that table to | |
75 | // the table cache. | |
76 | for (int i = 0; i < 2; ++i) { | |
77 | Get(ToString(i * 100 + 0)); | |
78 | } | |
79 | ||
80 | VerifyTableProperties(db_, 10 + 11 + 12 + 13); | |
81 | ||
82 | // 3. Put all tables to table cache | |
83 | Reopen(options); | |
84 | // fetch key from 1st and 2nd table, which will internally place that table to | |
85 | // the table cache. | |
86 | for (int i = 0; i < 4; ++i) { | |
87 | Get(ToString(i * 100 + 0)); | |
88 | } | |
89 | VerifyTableProperties(db_, 10 + 11 + 12 + 13); | |
90 | } | |
91 | ||
92 | TablePropertiesCollection | |
93 | DBTablePropertiesTest::TestGetPropertiesOfTablesInRange( | |
94 | std::vector<Range> ranges, std::size_t* num_properties, | |
95 | std::size_t* num_files) { | |
96 | ||
97 | // Since we deref zero element in the vector it can not be empty | |
98 | // otherwise we pass an address to some random memory | |
99 | EXPECT_GT(ranges.size(), 0U); | |
100 | // run the query | |
101 | TablePropertiesCollection props; | |
102 | EXPECT_OK(db_->GetPropertiesOfTablesInRange( | |
103 | db_->DefaultColumnFamily(), &ranges[0], ranges.size(), &props)); | |
104 | ||
105 | // Make sure that we've received properties for those and for those files | |
106 | // only which fall within requested ranges | |
107 | std::vector<LiveFileMetaData> vmd; | |
108 | db_->GetLiveFilesMetaData(&vmd); | |
109 | for (auto& md : vmd) { | |
110 | std::string fn = md.db_path + md.name; | |
111 | bool in_range = false; | |
112 | for (auto& r : ranges) { | |
113 | // smallestkey < limit && largestkey >= start | |
114 | if (r.limit.compare(md.smallestkey) >= 0 && | |
115 | r.start.compare(md.largestkey) <= 0) { | |
116 | in_range = true; | |
117 | EXPECT_GT(props.count(fn), 0); | |
118 | } | |
119 | } | |
120 | if (!in_range) { | |
121 | EXPECT_EQ(props.count(fn), 0); | |
122 | } | |
123 | } | |
124 | ||
125 | if (num_properties) { | |
126 | *num_properties = props.size(); | |
127 | } | |
128 | ||
129 | if (num_files) { | |
130 | *num_files = vmd.size(); | |
131 | } | |
132 | return props; | |
133 | } | |
134 | ||
135 | TEST_F(DBTablePropertiesTest, GetPropertiesOfTablesInRange) { | |
136 | // Fixed random sead | |
137 | Random rnd(301); | |
138 | ||
139 | Options options; | |
140 | options.create_if_missing = true; | |
141 | options.write_buffer_size = 4096; | |
f67539c2 | 142 | options.max_write_buffer_number = 2; |
7c673cae FG |
143 | options.level0_file_num_compaction_trigger = 2; |
144 | options.level0_slowdown_writes_trigger = 2; | |
f67539c2 | 145 | options.level0_stop_writes_trigger = 2; |
7c673cae | 146 | options.target_file_size_base = 2048; |
f67539c2 | 147 | options.max_bytes_for_level_base = 40960; |
7c673cae FG |
148 | options.max_bytes_for_level_multiplier = 4; |
149 | options.hard_pending_compaction_bytes_limit = 16 * 1024; | |
150 | options.num_levels = 8; | |
151 | options.env = env_; | |
152 | ||
153 | DestroyAndReopen(options); | |
154 | ||
155 | // build a decent LSM | |
156 | for (int i = 0; i < 10000; i++) { | |
157 | ASSERT_OK(Put(test::RandomKey(&rnd, 5), RandomString(&rnd, 102))); | |
158 | } | |
159 | Flush(); | |
160 | dbfull()->TEST_WaitForCompact(); | |
161 | if (NumTableFilesAtLevel(0) == 0) { | |
162 | ASSERT_OK(Put(test::RandomKey(&rnd, 5), RandomString(&rnd, 102))); | |
163 | Flush(); | |
164 | } | |
165 | ||
166 | db_->PauseBackgroundWork(); | |
167 | ||
168 | // Ensure that we have at least L0, L1 and L2 | |
169 | ASSERT_GT(NumTableFilesAtLevel(0), 0); | |
170 | ASSERT_GT(NumTableFilesAtLevel(1), 0); | |
171 | ASSERT_GT(NumTableFilesAtLevel(2), 0); | |
172 | ||
173 | // Query the largest range | |
174 | std::size_t num_properties, num_files; | |
175 | TestGetPropertiesOfTablesInRange( | |
176 | {Range(test::RandomKey(&rnd, 5, test::RandomKeyType::SMALLEST), | |
177 | test::RandomKey(&rnd, 5, test::RandomKeyType::LARGEST))}, | |
178 | &num_properties, &num_files); | |
179 | ASSERT_EQ(num_properties, num_files); | |
180 | ||
181 | // Query the empty range | |
182 | TestGetPropertiesOfTablesInRange( | |
183 | {Range(test::RandomKey(&rnd, 5, test::RandomKeyType::LARGEST), | |
184 | test::RandomKey(&rnd, 5, test::RandomKeyType::SMALLEST))}, | |
185 | &num_properties, &num_files); | |
186 | ASSERT_GT(num_files, 0); | |
187 | ASSERT_EQ(num_properties, 0); | |
188 | ||
189 | // Query the middle rangee | |
190 | TestGetPropertiesOfTablesInRange( | |
191 | {Range(test::RandomKey(&rnd, 5, test::RandomKeyType::MIDDLE), | |
192 | test::RandomKey(&rnd, 5, test::RandomKeyType::LARGEST))}, | |
193 | &num_properties, &num_files); | |
194 | ASSERT_GT(num_files, 0); | |
195 | ASSERT_GT(num_files, num_properties); | |
196 | ASSERT_GT(num_properties, 0); | |
197 | ||
198 | // Query a bunch of random ranges | |
199 | for (int j = 0; j < 100; j++) { | |
200 | // create a bunch of ranges | |
201 | std::vector<std::string> random_keys; | |
202 | // Random returns numbers with zero included | |
203 | // when we pass empty ranges TestGetPropertiesOfTablesInRange() | |
204 | // derefs random memory in the empty ranges[0] | |
205 | // so want to be greater than zero and even since | |
206 | // the below loop requires that random_keys.size() to be even. | |
207 | auto n = 2 * (rnd.Uniform(50) + 1); | |
208 | ||
209 | for (uint32_t i = 0; i < n; ++i) { | |
210 | random_keys.push_back(test::RandomKey(&rnd, 5)); | |
211 | } | |
212 | ||
213 | ASSERT_GT(random_keys.size(), 0U); | |
214 | ASSERT_EQ((random_keys.size() % 2), 0U); | |
215 | ||
216 | std::vector<Range> ranges; | |
217 | auto it = random_keys.begin(); | |
218 | while (it != random_keys.end()) { | |
219 | ranges.push_back(Range(*it, *(it + 1))); | |
220 | it += 2; | |
221 | } | |
222 | ||
223 | TestGetPropertiesOfTablesInRange(std::move(ranges)); | |
224 | } | |
225 | } | |
226 | ||
227 | TEST_F(DBTablePropertiesTest, GetColumnFamilyNameProperty) { | |
228 | std::string kExtraCfName = "pikachu"; | |
229 | CreateAndReopenWithCF({kExtraCfName}, CurrentOptions()); | |
230 | ||
231 | // Create one table per CF, then verify it was created with the column family | |
232 | // name property. | |
f67539c2 | 233 | for (uint32_t cf = 0; cf < 2; ++cf) { |
7c673cae FG |
234 | Put(cf, "key", "val"); |
235 | Flush(cf); | |
236 | ||
237 | TablePropertiesCollection fname_to_props; | |
238 | ASSERT_OK(db_->GetPropertiesOfAllTables(handles_[cf], &fname_to_props)); | |
239 | ASSERT_EQ(1U, fname_to_props.size()); | |
240 | ||
241 | std::string expected_cf_name; | |
242 | if (cf > 0) { | |
243 | expected_cf_name = kExtraCfName; | |
244 | } else { | |
245 | expected_cf_name = kDefaultColumnFamilyName; | |
246 | } | |
247 | ASSERT_EQ(expected_cf_name, | |
248 | fname_to_props.begin()->second->column_family_name); | |
249 | ASSERT_EQ(cf, static_cast<uint32_t>( | |
250 | fname_to_props.begin()->second->column_family_id)); | |
251 | } | |
252 | } | |
253 | ||
11fdf7f2 TL |
254 | TEST_F(DBTablePropertiesTest, DeletionTriggeredCompactionMarking) { |
255 | int kNumKeys = 1000; | |
256 | int kWindowSize = 100; | |
257 | int kNumDelsTrigger = 90; | |
258 | std::shared_ptr<TablePropertiesCollectorFactory> compact_on_del = | |
259 | NewCompactOnDeletionCollectorFactory(kWindowSize, kNumDelsTrigger); | |
260 | ||
261 | Options opts = CurrentOptions(); | |
262 | opts.table_properties_collector_factories.emplace_back(compact_on_del); | |
263 | Reopen(opts); | |
264 | ||
265 | // add an L1 file to prevent tombstones from dropping due to obsolescence | |
266 | // during flush | |
267 | Put(Key(0), "val"); | |
268 | Flush(); | |
269 | MoveFilesToLevel(1); | |
270 | ||
271 | for (int i = 0; i < kNumKeys; ++i) { | |
272 | if (i >= kNumKeys - kWindowSize && | |
273 | i < kNumKeys - kWindowSize + kNumDelsTrigger) { | |
274 | Delete(Key(i)); | |
275 | } else { | |
276 | Put(Key(i), "val"); | |
277 | } | |
278 | } | |
279 | Flush(); | |
280 | ||
281 | dbfull()->TEST_WaitForCompact(); | |
282 | ASSERT_EQ(0, NumTableFilesAtLevel(0)); | |
283 | ASSERT_GT(NumTableFilesAtLevel(1), 0); | |
284 | ||
285 | // Change the window size and deletion trigger and ensure new values take | |
286 | // effect | |
287 | kWindowSize = 50; | |
288 | kNumDelsTrigger = 40; | |
289 | static_cast<CompactOnDeletionCollectorFactory*> | |
290 | (compact_on_del.get())->SetWindowSize(kWindowSize); | |
291 | static_cast<CompactOnDeletionCollectorFactory*> | |
292 | (compact_on_del.get())->SetDeletionTrigger(kNumDelsTrigger); | |
293 | for (int i = 0; i < kNumKeys; ++i) { | |
294 | if (i >= kNumKeys - kWindowSize && | |
295 | i < kNumKeys - kWindowSize + kNumDelsTrigger) { | |
296 | Delete(Key(i)); | |
297 | } else { | |
298 | Put(Key(i), "val"); | |
299 | } | |
300 | } | |
301 | Flush(); | |
302 | ||
303 | dbfull()->TEST_WaitForCompact(); | |
304 | ASSERT_EQ(0, NumTableFilesAtLevel(0)); | |
305 | ASSERT_GT(NumTableFilesAtLevel(1), 0); | |
306 | ||
307 | // Change the window size to disable delete triggered compaction | |
308 | kWindowSize = 0; | |
309 | static_cast<CompactOnDeletionCollectorFactory*> | |
310 | (compact_on_del.get())->SetWindowSize(kWindowSize); | |
311 | static_cast<CompactOnDeletionCollectorFactory*> | |
312 | (compact_on_del.get())->SetDeletionTrigger(kNumDelsTrigger); | |
313 | for (int i = 0; i < kNumKeys; ++i) { | |
314 | if (i >= kNumKeys - kWindowSize && | |
315 | i < kNumKeys - kWindowSize + kNumDelsTrigger) { | |
316 | Delete(Key(i)); | |
317 | } else { | |
318 | Put(Key(i), "val"); | |
319 | } | |
320 | } | |
321 | Flush(); | |
322 | ||
323 | dbfull()->TEST_WaitForCompact(); | |
324 | ASSERT_EQ(1, NumTableFilesAtLevel(0)); | |
325 | ||
326 | } | |
327 | ||
f67539c2 | 328 | } // namespace ROCKSDB_NAMESPACE |
7c673cae FG |
329 | |
330 | #endif // ROCKSDB_LITE | |
331 | ||
332 | int main(int argc, char** argv) { | |
f67539c2 | 333 | ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); |
7c673cae FG |
334 | ::testing::InitGoogleTest(&argc, argv); |
335 | return RUN_ALL_TESTS(); | |
336 | } |