]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. |
2 | // This source code is licensed under the BSD-style license found in the | |
3 | // LICENSE file in the root directory of this source tree. An additional grant | |
4 | // of patent rights can be found in the PATENTS file in the same directory. | |
5 | // | |
6 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. | |
7 | // Use of this source code is governed by a BSD-style license that can be | |
8 | // found in the LICENSE file. See the AUTHORS file for names of contributors. | |
9 | ||
10 | #ifndef ROCKSDB_LITE | |
11 | ||
12 | #include <stdlib.h> | |
13 | #include <map> | |
14 | #include <string> | |
15 | #include <vector> | |
16 | #include "db/db_impl.h" | |
17 | #include "db/version_set.h" | |
18 | #include "db/write_batch_internal.h" | |
19 | #include "rocksdb/db.h" | |
20 | #include "rocksdb/env.h" | |
21 | #include "rocksdb/transaction_log.h" | |
22 | #include "util/filename.h" | |
23 | #include "util/string_util.h" | |
24 | #include "util/sync_point.h" | |
25 | #include "util/testharness.h" | |
26 | #include "util/testutil.h" | |
27 | ||
28 | namespace rocksdb { | |
29 | ||
30 | class DeleteFileTest : public testing::Test { | |
31 | public: | |
32 | std::string dbname_; | |
33 | Options options_; | |
34 | DB* db_; | |
35 | Env* env_; | |
36 | int numlevels_; | |
37 | ||
38 | DeleteFileTest() { | |
39 | db_ = nullptr; | |
40 | env_ = Env::Default(); | |
41 | options_.delete_obsolete_files_period_micros = 0; // always do full purge | |
42 | options_.enable_thread_tracking = true; | |
43 | options_.write_buffer_size = 1024*1024*1000; | |
44 | options_.target_file_size_base = 1024*1024*1000; | |
45 | options_.max_bytes_for_level_base = 1024*1024*1000; | |
46 | options_.WAL_ttl_seconds = 300; // Used to test log files | |
47 | options_.WAL_size_limit_MB = 1024; // Used to test log files | |
48 | dbname_ = test::TmpDir() + "/deletefile_test"; | |
49 | options_.wal_dir = dbname_ + "/wal_files"; | |
50 | ||
51 | // clean up all the files that might have been there before | |
52 | std::vector<std::string> old_files; | |
53 | env_->GetChildren(dbname_, &old_files); | |
54 | for (auto file : old_files) { | |
55 | env_->DeleteFile(dbname_ + "/" + file); | |
56 | } | |
57 | env_->GetChildren(options_.wal_dir, &old_files); | |
58 | for (auto file : old_files) { | |
59 | env_->DeleteFile(options_.wal_dir + "/" + file); | |
60 | } | |
61 | ||
62 | DestroyDB(dbname_, options_); | |
63 | numlevels_ = 7; | |
64 | EXPECT_OK(ReopenDB(true)); | |
65 | } | |
66 | ||
67 | Status ReopenDB(bool create) { | |
68 | delete db_; | |
69 | if (create) { | |
70 | DestroyDB(dbname_, options_); | |
71 | } | |
72 | db_ = nullptr; | |
73 | options_.create_if_missing = create; | |
74 | return DB::Open(options_, dbname_, &db_); | |
75 | } | |
76 | ||
77 | void CloseDB() { | |
78 | delete db_; | |
79 | db_ = nullptr; | |
80 | } | |
81 | ||
82 | void AddKeys(int numkeys, int startkey = 0) { | |
83 | WriteOptions options; | |
84 | options.sync = false; | |
85 | ReadOptions roptions; | |
86 | for (int i = startkey; i < (numkeys + startkey) ; i++) { | |
87 | std::string temp = ToString(i); | |
88 | Slice key(temp); | |
89 | Slice value(temp); | |
90 | ASSERT_OK(db_->Put(options, key, value)); | |
91 | } | |
92 | } | |
93 | ||
94 | int numKeysInLevels( | |
95 | std::vector<LiveFileMetaData> &metadata, | |
96 | std::vector<int> *keysperlevel = nullptr) { | |
97 | ||
98 | if (keysperlevel != nullptr) { | |
99 | keysperlevel->resize(numlevels_); | |
100 | } | |
101 | ||
102 | int numKeys = 0; | |
103 | for (size_t i = 0; i < metadata.size(); i++) { | |
104 | int startkey = atoi(metadata[i].smallestkey.c_str()); | |
105 | int endkey = atoi(metadata[i].largestkey.c_str()); | |
106 | int numkeysinfile = (endkey - startkey + 1); | |
107 | numKeys += numkeysinfile; | |
108 | if (keysperlevel != nullptr) { | |
109 | (*keysperlevel)[(int)metadata[i].level] += numkeysinfile; | |
110 | } | |
111 | fprintf(stderr, "level %d name %s smallest %s largest %s\n", | |
112 | metadata[i].level, metadata[i].name.c_str(), | |
113 | metadata[i].smallestkey.c_str(), | |
114 | metadata[i].largestkey.c_str()); | |
115 | } | |
116 | return numKeys; | |
117 | } | |
118 | ||
119 | void CreateTwoLevels() { | |
120 | AddKeys(50000, 10000); | |
121 | DBImpl* dbi = reinterpret_cast<DBImpl*>(db_); | |
122 | ASSERT_OK(dbi->TEST_FlushMemTable()); | |
123 | ASSERT_OK(dbi->TEST_WaitForFlushMemTable()); | |
124 | for (int i = 0; i < 2; ++i) { | |
125 | ASSERT_OK(dbi->TEST_CompactRange(i, nullptr, nullptr)); | |
126 | } | |
127 | ||
128 | AddKeys(50000, 10000); | |
129 | ASSERT_OK(dbi->TEST_FlushMemTable()); | |
130 | ASSERT_OK(dbi->TEST_WaitForFlushMemTable()); | |
131 | ASSERT_OK(dbi->TEST_CompactRange(0, nullptr, nullptr)); | |
132 | } | |
133 | ||
134 | void CheckFileTypeCounts(std::string& dir, | |
135 | int required_log, | |
136 | int required_sst, | |
137 | int required_manifest) { | |
138 | std::vector<std::string> filenames; | |
139 | env_->GetChildren(dir, &filenames); | |
140 | ||
141 | int log_cnt = 0, sst_cnt = 0, manifest_cnt = 0; | |
142 | for (auto file : filenames) { | |
143 | uint64_t number; | |
144 | FileType type; | |
145 | if (ParseFileName(file, &number, &type)) { | |
146 | log_cnt += (type == kLogFile); | |
147 | sst_cnt += (type == kTableFile); | |
148 | manifest_cnt += (type == kDescriptorFile); | |
149 | } | |
150 | } | |
151 | ASSERT_EQ(required_log, log_cnt); | |
152 | ASSERT_EQ(required_sst, sst_cnt); | |
153 | ASSERT_EQ(required_manifest, manifest_cnt); | |
154 | } | |
155 | ||
156 | static void DoSleep(void* arg) { | |
157 | auto test = reinterpret_cast<DeleteFileTest*>(arg); | |
158 | test->env_->SleepForMicroseconds(2 * 1000 * 1000); | |
159 | } | |
160 | ||
161 | // An empty job to guard all jobs are processed | |
162 | static void GuardFinish(void* arg) { | |
163 | TEST_SYNC_POINT("DeleteFileTest::GuardFinish"); | |
164 | } | |
165 | }; | |
166 | ||
167 | TEST_F(DeleteFileTest, AddKeysAndQueryLevels) { | |
168 | CreateTwoLevels(); | |
169 | std::vector<LiveFileMetaData> metadata; | |
170 | db_->GetLiveFilesMetaData(&metadata); | |
171 | ||
172 | std::string level1file = ""; | |
173 | int level1keycount = 0; | |
174 | std::string level2file = ""; | |
175 | int level2keycount = 0; | |
176 | int level1index = 0; | |
177 | int level2index = 1; | |
178 | ||
179 | ASSERT_EQ((int)metadata.size(), 2); | |
180 | if (metadata[0].level == 2) { | |
181 | level1index = 1; | |
182 | level2index = 0; | |
183 | } | |
184 | ||
185 | level1file = metadata[level1index].name; | |
186 | int startkey = atoi(metadata[level1index].smallestkey.c_str()); | |
187 | int endkey = atoi(metadata[level1index].largestkey.c_str()); | |
188 | level1keycount = (endkey - startkey + 1); | |
189 | level2file = metadata[level2index].name; | |
190 | startkey = atoi(metadata[level2index].smallestkey.c_str()); | |
191 | endkey = atoi(metadata[level2index].largestkey.c_str()); | |
192 | level2keycount = (endkey - startkey + 1); | |
193 | ||
194 | // COntrolled setup. Levels 1 and 2 should both have 50K files. | |
195 | // This is a little fragile as it depends on the current | |
196 | // compaction heuristics. | |
197 | ASSERT_EQ(level1keycount, 50000); | |
198 | ASSERT_EQ(level2keycount, 50000); | |
199 | ||
200 | Status status = db_->DeleteFile("0.sst"); | |
201 | ASSERT_TRUE(status.IsInvalidArgument()); | |
202 | ||
203 | // intermediate level files cannot be deleted. | |
204 | status = db_->DeleteFile(level1file); | |
205 | ASSERT_TRUE(status.IsInvalidArgument()); | |
206 | ||
207 | // Lowest level file deletion should succeed. | |
208 | ASSERT_OK(db_->DeleteFile(level2file)); | |
209 | ||
210 | CloseDB(); | |
211 | } | |
212 | ||
213 | TEST_F(DeleteFileTest, PurgeObsoleteFilesTest) { | |
214 | CreateTwoLevels(); | |
215 | // there should be only one (empty) log file because CreateTwoLevels() | |
216 | // flushes the memtables to disk | |
217 | CheckFileTypeCounts(options_.wal_dir, 1, 0, 0); | |
218 | // 2 ssts, 1 manifest | |
219 | CheckFileTypeCounts(dbname_, 0, 2, 1); | |
220 | std::string first("0"), last("999999"); | |
221 | CompactRangeOptions compact_options; | |
222 | compact_options.change_level = true; | |
223 | compact_options.target_level = 2; | |
224 | Slice first_slice(first), last_slice(last); | |
225 | db_->CompactRange(compact_options, &first_slice, &last_slice); | |
226 | // 1 sst after compaction | |
227 | CheckFileTypeCounts(dbname_, 0, 1, 1); | |
228 | ||
229 | // this time, we keep an iterator alive | |
230 | ReopenDB(true); | |
231 | Iterator *itr = 0; | |
232 | CreateTwoLevels(); | |
233 | itr = db_->NewIterator(ReadOptions()); | |
234 | db_->CompactRange(compact_options, &first_slice, &last_slice); | |
235 | // 3 sst after compaction with live iterator | |
236 | CheckFileTypeCounts(dbname_, 0, 3, 1); | |
237 | delete itr; | |
238 | // 1 sst after iterator deletion | |
239 | CheckFileTypeCounts(dbname_, 0, 1, 1); | |
240 | ||
241 | CloseDB(); | |
242 | } | |
243 | ||
244 | TEST_F(DeleteFileTest, BackgroundPurgeTest) { | |
245 | std::string first("0"), last("999999"); | |
246 | CompactRangeOptions compact_options; | |
247 | compact_options.change_level = true; | |
248 | compact_options.target_level = 2; | |
249 | Slice first_slice(first), last_slice(last); | |
250 | ||
251 | // We keep an iterator alive | |
252 | Iterator* itr = 0; | |
253 | CreateTwoLevels(); | |
254 | ReadOptions options; | |
255 | options.background_purge_on_iterator_cleanup = true; | |
256 | itr = db_->NewIterator(options); | |
257 | db_->CompactRange(compact_options, &first_slice, &last_slice); | |
258 | // 3 sst after compaction with live iterator | |
259 | CheckFileTypeCounts(dbname_, 0, 3, 1); | |
260 | test::SleepingBackgroundTask sleeping_task_before; | |
261 | env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, | |
262 | &sleeping_task_before, Env::Priority::HIGH); | |
263 | delete itr; | |
264 | test::SleepingBackgroundTask sleeping_task_after; | |
265 | env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, | |
266 | &sleeping_task_after, Env::Priority::HIGH); | |
267 | ||
268 | // Make sure no purges are executed foreground | |
269 | CheckFileTypeCounts(dbname_, 0, 3, 1); | |
270 | sleeping_task_before.WakeUp(); | |
271 | sleeping_task_before.WaitUntilDone(); | |
272 | ||
273 | // Make sure all background purges are executed | |
274 | sleeping_task_after.WakeUp(); | |
275 | sleeping_task_after.WaitUntilDone(); | |
276 | // 1 sst after iterator deletion | |
277 | CheckFileTypeCounts(dbname_, 0, 1, 1); | |
278 | ||
279 | CloseDB(); | |
280 | } | |
281 | ||
282 | // This test is to reproduce a bug that read invalid ReadOption in iterator | |
283 | // cleanup function | |
284 | TEST_F(DeleteFileTest, BackgroundPurgeCopyOptions) { | |
285 | std::string first("0"), last("999999"); | |
286 | CompactRangeOptions compact_options; | |
287 | compact_options.change_level = true; | |
288 | compact_options.target_level = 2; | |
289 | Slice first_slice(first), last_slice(last); | |
290 | ||
291 | // We keep an iterator alive | |
292 | Iterator* itr = 0; | |
293 | CreateTwoLevels(); | |
294 | ReadOptions* options = new ReadOptions(); | |
295 | options->background_purge_on_iterator_cleanup = true; | |
296 | itr = db_->NewIterator(*options); | |
297 | // ReadOptions is deleted, but iterator cleanup function should not be | |
298 | // affected | |
299 | delete options; | |
300 | ||
301 | db_->CompactRange(compact_options, &first_slice, &last_slice); | |
302 | // 3 sst after compaction with live iterator | |
303 | CheckFileTypeCounts(dbname_, 0, 3, 1); | |
304 | delete itr; | |
305 | ||
306 | test::SleepingBackgroundTask sleeping_task_after; | |
307 | env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, | |
308 | &sleeping_task_after, Env::Priority::HIGH); | |
309 | ||
310 | // Make sure all background purges are executed | |
311 | sleeping_task_after.WakeUp(); | |
312 | sleeping_task_after.WaitUntilDone(); | |
313 | // 1 sst after iterator deletion | |
314 | CheckFileTypeCounts(dbname_, 0, 1, 1); | |
315 | ||
316 | CloseDB(); | |
317 | } | |
318 | ||
319 | TEST_F(DeleteFileTest, BackgroundPurgeTestMultipleJobs) { | |
320 | std::string first("0"), last("999999"); | |
321 | CompactRangeOptions compact_options; | |
322 | compact_options.change_level = true; | |
323 | compact_options.target_level = 2; | |
324 | Slice first_slice(first), last_slice(last); | |
325 | ||
326 | // We keep an iterator alive | |
327 | CreateTwoLevels(); | |
328 | ReadOptions options; | |
329 | options.background_purge_on_iterator_cleanup = true; | |
330 | Iterator* itr1 = db_->NewIterator(options); | |
331 | CreateTwoLevels(); | |
332 | Iterator* itr2 = db_->NewIterator(options); | |
333 | db_->CompactRange(compact_options, &first_slice, &last_slice); | |
334 | // 5 sst files after 2 compactions with 2 live iterators | |
335 | CheckFileTypeCounts(dbname_, 0, 5, 1); | |
336 | ||
337 | // ~DBImpl should wait until all BGWorkPurge are finished | |
338 | rocksdb::SyncPoint::GetInstance()->LoadDependency( | |
339 | {{"DBImpl::~DBImpl:WaitJob", "DBImpl::BGWorkPurge"}, | |
340 | {"DeleteFileTest::GuardFinish", | |
341 | "DeleteFileTest::BackgroundPurgeTestMultipleJobs:DBClose"}}); | |
342 | rocksdb::SyncPoint::GetInstance()->EnableProcessing(); | |
343 | ||
344 | delete itr1; | |
345 | env_->Schedule(&DeleteFileTest::DoSleep, this, Env::Priority::HIGH); | |
346 | delete itr2; | |
347 | env_->Schedule(&DeleteFileTest::GuardFinish, nullptr, Env::Priority::HIGH); | |
348 | CloseDB(); | |
349 | ||
350 | TEST_SYNC_POINT("DeleteFileTest::BackgroundPurgeTestMultipleJobs:DBClose"); | |
351 | // 1 sst after iterator deletion | |
352 | CheckFileTypeCounts(dbname_, 0, 1, 1); | |
353 | rocksdb::SyncPoint::GetInstance()->DisableProcessing(); | |
354 | } | |
355 | ||
356 | TEST_F(DeleteFileTest, DeleteFileWithIterator) { | |
357 | CreateTwoLevels(); | |
358 | ReadOptions options; | |
359 | Iterator* it = db_->NewIterator(options); | |
360 | std::vector<LiveFileMetaData> metadata; | |
361 | db_->GetLiveFilesMetaData(&metadata); | |
362 | ||
363 | std::string level2file = ""; | |
364 | ||
365 | ASSERT_EQ((int)metadata.size(), 2); | |
366 | if (metadata[0].level == 1) { | |
367 | level2file = metadata[1].name; | |
368 | } else { | |
369 | level2file = metadata[0].name; | |
370 | } | |
371 | ||
372 | Status status = db_->DeleteFile(level2file); | |
373 | fprintf(stdout, "Deletion status %s: %s\n", | |
374 | level2file.c_str(), status.ToString().c_str()); | |
375 | ASSERT_TRUE(status.ok()); | |
376 | it->SeekToFirst(); | |
377 | int numKeysIterated = 0; | |
378 | while(it->Valid()) { | |
379 | numKeysIterated++; | |
380 | it->Next(); | |
381 | } | |
382 | ASSERT_EQ(numKeysIterated, 50000); | |
383 | delete it; | |
384 | CloseDB(); | |
385 | } | |
386 | ||
387 | TEST_F(DeleteFileTest, DeleteLogFiles) { | |
388 | AddKeys(10, 0); | |
389 | VectorLogPtr logfiles; | |
390 | db_->GetSortedWalFiles(logfiles); | |
391 | ASSERT_GT(logfiles.size(), 0UL); | |
392 | // Take the last log file which is expected to be alive and try to delete it | |
393 | // Should not succeed because live logs are not allowed to be deleted | |
394 | std::unique_ptr<LogFile> alive_log = std::move(logfiles.back()); | |
395 | ASSERT_EQ(alive_log->Type(), kAliveLogFile); | |
396 | ASSERT_OK(env_->FileExists(options_.wal_dir + "/" + alive_log->PathName())); | |
397 | fprintf(stdout, "Deleting alive log file %s\n", | |
398 | alive_log->PathName().c_str()); | |
399 | ASSERT_TRUE(!db_->DeleteFile(alive_log->PathName()).ok()); | |
400 | ASSERT_OK(env_->FileExists(options_.wal_dir + "/" + alive_log->PathName())); | |
401 | logfiles.clear(); | |
402 | ||
403 | // Call Flush to bring about a new working log file and add more keys | |
404 | // Call Flush again to flush out memtable and move alive log to archived log | |
405 | // and try to delete the archived log file | |
406 | FlushOptions fopts; | |
407 | db_->Flush(fopts); | |
408 | AddKeys(10, 0); | |
409 | db_->Flush(fopts); | |
410 | db_->GetSortedWalFiles(logfiles); | |
411 | ASSERT_GT(logfiles.size(), 0UL); | |
412 | std::unique_ptr<LogFile> archived_log = std::move(logfiles.front()); | |
413 | ASSERT_EQ(archived_log->Type(), kArchivedLogFile); | |
414 | ASSERT_OK( | |
415 | env_->FileExists(options_.wal_dir + "/" + archived_log->PathName())); | |
416 | fprintf(stdout, "Deleting archived log file %s\n", | |
417 | archived_log->PathName().c_str()); | |
418 | ASSERT_OK(db_->DeleteFile(archived_log->PathName())); | |
419 | ASSERT_EQ(Status::NotFound(), env_->FileExists(options_.wal_dir + "/" + | |
420 | archived_log->PathName())); | |
421 | CloseDB(); | |
422 | } | |
423 | ||
424 | TEST_F(DeleteFileTest, DeleteNonDefaultColumnFamily) { | |
425 | CloseDB(); | |
426 | DBOptions db_options; | |
427 | db_options.create_if_missing = true; | |
428 | db_options.create_missing_column_families = true; | |
429 | std::vector<ColumnFamilyDescriptor> column_families; | |
430 | column_families.emplace_back(); | |
431 | column_families.emplace_back("new_cf", ColumnFamilyOptions()); | |
432 | ||
433 | std::vector<rocksdb::ColumnFamilyHandle*> handles; | |
434 | rocksdb::DB* db; | |
435 | ASSERT_OK(DB::Open(db_options, dbname_, column_families, &handles, &db)); | |
436 | ||
437 | Random rnd(5); | |
438 | for (int i = 0; i < 1000; ++i) { | |
439 | ASSERT_OK(db->Put(WriteOptions(), handles[1], test::RandomKey(&rnd, 10), | |
440 | test::RandomKey(&rnd, 10))); | |
441 | } | |
442 | ASSERT_OK(db->Flush(FlushOptions(), handles[1])); | |
443 | for (int i = 0; i < 1000; ++i) { | |
444 | ASSERT_OK(db->Put(WriteOptions(), handles[1], test::RandomKey(&rnd, 10), | |
445 | test::RandomKey(&rnd, 10))); | |
446 | } | |
447 | ASSERT_OK(db->Flush(FlushOptions(), handles[1])); | |
448 | ||
449 | std::vector<LiveFileMetaData> metadata; | |
450 | db->GetLiveFilesMetaData(&metadata); | |
451 | ASSERT_EQ(2U, metadata.size()); | |
452 | ASSERT_EQ("new_cf", metadata[0].column_family_name); | |
453 | ASSERT_EQ("new_cf", metadata[1].column_family_name); | |
454 | auto old_file = metadata[0].smallest_seqno < metadata[1].smallest_seqno | |
455 | ? metadata[0].name | |
456 | : metadata[1].name; | |
457 | auto new_file = metadata[0].smallest_seqno > metadata[1].smallest_seqno | |
458 | ? metadata[0].name | |
459 | : metadata[1].name; | |
460 | ASSERT_TRUE(db->DeleteFile(new_file).IsInvalidArgument()); | |
461 | ASSERT_OK(db->DeleteFile(old_file)); | |
462 | ||
463 | { | |
464 | std::unique_ptr<Iterator> itr(db->NewIterator(ReadOptions(), handles[1])); | |
465 | int count = 0; | |
466 | for (itr->SeekToFirst(); itr->Valid(); itr->Next()) { | |
467 | ASSERT_OK(itr->status()); | |
468 | ++count; | |
469 | } | |
470 | ASSERT_EQ(count, 1000); | |
471 | } | |
472 | ||
473 | delete handles[0]; | |
474 | delete handles[1]; | |
475 | delete db; | |
476 | ||
477 | ASSERT_OK(DB::Open(db_options, dbname_, column_families, &handles, &db)); | |
478 | { | |
479 | std::unique_ptr<Iterator> itr(db->NewIterator(ReadOptions(), handles[1])); | |
480 | int count = 0; | |
481 | for (itr->SeekToFirst(); itr->Valid(); itr->Next()) { | |
482 | ASSERT_OK(itr->status()); | |
483 | ++count; | |
484 | } | |
485 | ASSERT_EQ(count, 1000); | |
486 | } | |
487 | ||
488 | delete handles[0]; | |
489 | delete handles[1]; | |
490 | delete db; | |
491 | } | |
492 | ||
493 | } //namespace rocksdb | |
494 | ||
495 | int main(int argc, char** argv) { | |
496 | ::testing::InitGoogleTest(&argc, argv); | |
497 | return RUN_ALL_TESTS(); | |
498 | } | |
499 | ||
500 | #else | |
501 | #include <stdio.h> | |
502 | ||
503 | int main(int argc, char** argv) { | |
504 | fprintf(stderr, | |
505 | "SKIPPED as DBImpl::DeleteFile is not supported in ROCKSDB_LITE\n"); | |
506 | return 0; | |
507 | } | |
508 | ||
509 | #endif // !ROCKSDB_LITE |