1 // Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
2 // This source code is licensed under both the GPLv2 (found in the
3 // COPYING file in the root directory) and Apache 2.0 License
4 // (found in the LICENSE.Apache file in the root directory).
6 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
7 // Use of this source code is governed by a BSD-style license that can be
8 // found in the LICENSE file. See the AUTHORS file for names of contributors.
18 #include "db/db_impl/db_impl.h"
19 #include "db/db_test_util.h"
20 #include "db/version_set.h"
21 #include "db/write_batch_internal.h"
22 #include "file/filename.h"
23 #include "port/stack_trace.h"
24 #include "rocksdb/db.h"
25 #include "rocksdb/env.h"
26 #include "rocksdb/transaction_log.h"
27 #include "test_util/sync_point.h"
28 #include "test_util/testharness.h"
29 #include "test_util/testutil.h"
30 #include "util/string_util.h"
32 namespace ROCKSDB_NAMESPACE
{
34 class DeleteFileTest
: public DBTestBase
{
37 const std::string wal_dir_
;
40 : DBTestBase("deletefile_test", /*env_do_fsync=*/true),
42 wal_dir_(dbname_
+ "/wal_files") {}
44 void SetOptions(Options
* options
) {
45 ASSERT_NE(options
, nullptr);
46 options
->delete_obsolete_files_period_micros
= 0; // always do full purge
47 options
->enable_thread_tracking
= true;
48 options
->write_buffer_size
= 1024 * 1024 * 1000;
49 options
->target_file_size_base
= 1024 * 1024 * 1000;
50 options
->max_bytes_for_level_base
= 1024 * 1024 * 1000;
51 options
->WAL_ttl_seconds
= 300; // Used to test log files
52 options
->WAL_size_limit_MB
= 1024; // Used to test log files
53 options
->wal_dir
= wal_dir_
;
56 void AddKeys(int numkeys
, int startkey
= 0) {
60 for (int i
= startkey
; i
< (numkeys
+ startkey
); i
++) {
61 std::string temp
= std::to_string(i
);
64 ASSERT_OK(db_
->Put(options
, key
, value
));
68 int numKeysInLevels(std::vector
<LiveFileMetaData
>& metadata
,
69 std::vector
<int>* keysperlevel
= nullptr) {
70 if (keysperlevel
!= nullptr) {
71 keysperlevel
->resize(numlevels_
);
75 for (size_t i
= 0; i
< metadata
.size(); i
++) {
76 int startkey
= atoi(metadata
[i
].smallestkey
.c_str());
77 int endkey
= atoi(metadata
[i
].largestkey
.c_str());
78 int numkeysinfile
= (endkey
- startkey
+ 1);
79 numKeys
+= numkeysinfile
;
80 if (keysperlevel
!= nullptr) {
81 (*keysperlevel
)[(int)metadata
[i
].level
] += numkeysinfile
;
83 fprintf(stderr
, "level %d name %s smallest %s largest %s\n",
84 metadata
[i
].level
, metadata
[i
].name
.c_str(),
85 metadata
[i
].smallestkey
.c_str(), metadata
[i
].largestkey
.c_str());
90 void CreateTwoLevels() {
91 AddKeys(50000, 10000);
92 ASSERT_OK(dbfull()->TEST_FlushMemTable());
93 ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
94 for (int i
= 0; i
< 2; ++i
) {
95 ASSERT_OK(dbfull()->TEST_CompactRange(i
, nullptr, nullptr));
98 AddKeys(50000, 10000);
99 ASSERT_OK(dbfull()->TEST_FlushMemTable());
100 ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
101 ASSERT_OK(dbfull()->TEST_CompactRange(0, nullptr, nullptr));
104 void CheckFileTypeCounts(const std::string
& dir
, int required_log
,
105 int required_sst
, int required_manifest
) {
106 std::vector
<std::string
> filenames
;
107 ASSERT_OK(env_
->GetChildren(dir
, &filenames
));
109 int log_cnt
= 0, sst_cnt
= 0, manifest_cnt
= 0;
110 for (auto file
: filenames
) {
113 if (ParseFileName(file
, &number
, &type
)) {
114 log_cnt
+= (type
== kWalFile
);
115 sst_cnt
+= (type
== kTableFile
);
116 manifest_cnt
+= (type
== kDescriptorFile
);
119 if (required_log
>= 0) {
120 ASSERT_EQ(required_log
, log_cnt
);
122 if (required_sst
>= 0) {
123 ASSERT_EQ(required_sst
, sst_cnt
);
125 if (required_manifest
>= 0) {
126 ASSERT_EQ(required_manifest
, manifest_cnt
);
130 static void DoSleep(void* arg
) {
131 auto test
= reinterpret_cast<DeleteFileTest
*>(arg
);
132 test
->env_
->SleepForMicroseconds(2 * 1000 * 1000);
135 // An empty job to guard all jobs are processed
136 static void GuardFinish(void* /*arg*/) {
137 TEST_SYNC_POINT("DeleteFileTest::GuardFinish");
141 TEST_F(DeleteFileTest
, AddKeysAndQueryLevels
) {
142 Options options
= CurrentOptions();
143 SetOptions(&options
);
145 options
.create_if_missing
= true;
149 std::vector
<LiveFileMetaData
> metadata
;
150 db_
->GetLiveFilesMetaData(&metadata
);
152 std::string level1file
= "";
153 int level1keycount
= 0;
154 std::string level2file
= "";
155 int level2keycount
= 0;
159 ASSERT_EQ((int)metadata
.size(), 2);
160 if (metadata
[0].level
== 2) {
165 level1file
= metadata
[level1index
].name
;
166 int startkey
= atoi(metadata
[level1index
].smallestkey
.c_str());
167 int endkey
= atoi(metadata
[level1index
].largestkey
.c_str());
168 level1keycount
= (endkey
- startkey
+ 1);
169 level2file
= metadata
[level2index
].name
;
170 startkey
= atoi(metadata
[level2index
].smallestkey
.c_str());
171 endkey
= atoi(metadata
[level2index
].largestkey
.c_str());
172 level2keycount
= (endkey
- startkey
+ 1);
174 // COntrolled setup. Levels 1 and 2 should both have 50K files.
175 // This is a little fragile as it depends on the current
176 // compaction heuristics.
177 ASSERT_EQ(level1keycount
, 50000);
178 ASSERT_EQ(level2keycount
, 50000);
180 Status status
= db_
->DeleteFile("0.sst");
181 ASSERT_TRUE(status
.IsInvalidArgument());
183 // intermediate level files cannot be deleted.
184 status
= db_
->DeleteFile(level1file
);
185 ASSERT_TRUE(status
.IsInvalidArgument());
187 // Lowest level file deletion should succeed.
188 status
= db_
->DeleteFile(level2file
);
192 TEST_F(DeleteFileTest
, PurgeObsoleteFilesTest
) {
193 Options options
= CurrentOptions();
194 SetOptions(&options
);
196 options
.create_if_missing
= true;
200 // there should be only one (empty) log file because CreateTwoLevels()
201 // flushes the memtables to disk
202 CheckFileTypeCounts(wal_dir_
, 1, 0, 0);
203 // 2 ssts, 1 manifest
204 CheckFileTypeCounts(dbname_
, 0, 2, 1);
205 std::string
first("0"), last("999999");
206 CompactRangeOptions compact_options
;
207 compact_options
.change_level
= true;
208 compact_options
.target_level
= 2;
209 Slice
first_slice(first
), last_slice(last
);
210 ASSERT_OK(db_
->CompactRange(compact_options
, &first_slice
, &last_slice
));
211 // 1 sst after compaction
212 CheckFileTypeCounts(dbname_
, 0, 1, 1);
214 // this time, we keep an iterator alive
216 Iterator
* itr
= nullptr;
218 itr
= db_
->NewIterator(ReadOptions());
219 ASSERT_OK(itr
->status());
220 ASSERT_OK(db_
->CompactRange(compact_options
, &first_slice
, &last_slice
));
221 ASSERT_OK(itr
->status());
222 // 3 sst after compaction with live iterator
223 CheckFileTypeCounts(dbname_
, 0, 3, 1);
225 // 1 sst after iterator deletion
226 CheckFileTypeCounts(dbname_
, 0, 1, 1);
229 TEST_F(DeleteFileTest
, BackgroundPurgeIteratorTest
) {
230 Options options
= CurrentOptions();
231 SetOptions(&options
);
233 options
.create_if_missing
= true;
236 std::string
first("0"), last("999999");
237 CompactRangeOptions compact_options
;
238 compact_options
.change_level
= true;
239 compact_options
.target_level
= 2;
240 Slice
first_slice(first
), last_slice(last
);
242 // We keep an iterator alive
243 Iterator
* itr
= nullptr;
245 ReadOptions read_options
;
246 read_options
.background_purge_on_iterator_cleanup
= true;
247 itr
= db_
->NewIterator(read_options
);
248 ASSERT_OK(itr
->status());
249 ASSERT_OK(db_
->CompactRange(compact_options
, &first_slice
, &last_slice
));
250 // 3 sst after compaction with live iterator
251 CheckFileTypeCounts(dbname_
, 0, 3, 1);
252 test::SleepingBackgroundTask sleeping_task_before
;
253 env_
->Schedule(&test::SleepingBackgroundTask::DoSleepTask
,
254 &sleeping_task_before
, Env::Priority::HIGH
);
256 test::SleepingBackgroundTask sleeping_task_after
;
257 env_
->Schedule(&test::SleepingBackgroundTask::DoSleepTask
,
258 &sleeping_task_after
, Env::Priority::HIGH
);
260 // Make sure no purges are executed foreground
261 CheckFileTypeCounts(dbname_
, 0, 3, 1);
262 sleeping_task_before
.WakeUp();
263 sleeping_task_before
.WaitUntilDone();
265 // Make sure all background purges are executed
266 sleeping_task_after
.WakeUp();
267 sleeping_task_after
.WaitUntilDone();
268 // 1 sst after iterator deletion
269 CheckFileTypeCounts(dbname_
, 0, 1, 1);
272 TEST_F(DeleteFileTest
, PurgeDuringOpen
) {
273 Options options
= CurrentOptions();
274 CheckFileTypeCounts(dbname_
, -1, 0, -1);
276 std::unique_ptr
<WritableFile
> file
;
277 ASSERT_OK(options
.env
->NewWritableFile(dbname_
+ "/000002.sst", &file
,
279 ASSERT_OK(file
->Close());
280 CheckFileTypeCounts(dbname_
, -1, 1, -1);
281 options
.avoid_unnecessary_blocking_io
= false;
282 options
.create_if_missing
= false;
284 CheckFileTypeCounts(dbname_
, -1, 0, -1);
287 // test background purge
288 options
.avoid_unnecessary_blocking_io
= true;
289 options
.create_if_missing
= false;
290 ASSERT_OK(options
.env
->NewWritableFile(dbname_
+ "/000002.sst", &file
,
292 ASSERT_OK(file
->Close());
293 CheckFileTypeCounts(dbname_
, -1, 1, -1);
294 SyncPoint::GetInstance()->DisableProcessing();
295 SyncPoint::GetInstance()->ClearAllCallBacks();
296 SyncPoint::GetInstance()->LoadDependency(
297 {{"DeleteFileTest::PurgeDuringOpen:1", "DBImpl::BGWorkPurge:start"}});
298 SyncPoint::GetInstance()->EnableProcessing();
300 // the obsolete file is not deleted until the background purge job is ran
301 CheckFileTypeCounts(dbname_
, -1, 1, -1);
302 TEST_SYNC_POINT("DeleteFileTest::PurgeDuringOpen:1");
303 ASSERT_OK(dbfull()->TEST_WaitForPurge());
304 CheckFileTypeCounts(dbname_
, -1, 0, -1);
307 TEST_F(DeleteFileTest
, BackgroundPurgeCFDropTest
) {
308 Options options
= CurrentOptions();
309 SetOptions(&options
);
311 options
.create_if_missing
= true;
314 auto do_test
= [&](bool bg_purge
) {
315 ColumnFamilyOptions co
;
316 co
.max_write_buffer_size_to_maintain
=
317 static_cast<int64_t>(co
.write_buffer_size
);
320 ColumnFamilyHandle
* cfh
= nullptr;
322 ASSERT_OK(db_
->CreateColumnFamily(co
, "dropme", &cfh
));
324 ASSERT_OK(db_
->Put(wo
, cfh
, "pika", "chu"));
325 ASSERT_OK(db_
->Flush(fo
, cfh
));
326 // Expect 1 sst file.
327 CheckFileTypeCounts(dbname_
, 0, 1, 1);
329 ASSERT_OK(db_
->DropColumnFamily(cfh
));
330 // Still 1 file, it won't be deleted while ColumnFamilyHandle is alive.
331 CheckFileTypeCounts(dbname_
, 0, 1, 1);
334 test::SleepingBackgroundTask sleeping_task_after
;
335 env_
->Schedule(&test::SleepingBackgroundTask::DoSleepTask
,
336 &sleeping_task_after
, Env::Priority::HIGH
);
337 // If background purge is enabled, the file should still be there.
338 CheckFileTypeCounts(dbname_
, 0, bg_purge
? 1 : 0, 1);
339 TEST_SYNC_POINT("DeleteFileTest::BackgroundPurgeCFDropTest:1");
341 // Execute background purges.
342 sleeping_task_after
.WakeUp();
343 sleeping_task_after
.WaitUntilDone();
344 // The file should have been deleted.
345 CheckFileTypeCounts(dbname_
, 0, 0, 1);
349 SCOPED_TRACE("avoid_unnecessary_blocking_io = false");
353 options
.avoid_unnecessary_blocking_io
= true;
354 options
.create_if_missing
= false;
356 ASSERT_OK(dbfull()->TEST_WaitForPurge());
358 SyncPoint::GetInstance()->DisableProcessing();
359 SyncPoint::GetInstance()->ClearAllCallBacks();
360 SyncPoint::GetInstance()->LoadDependency(
361 {{"DeleteFileTest::BackgroundPurgeCFDropTest:1",
362 "DBImpl::BGWorkPurge:start"}});
363 SyncPoint::GetInstance()->EnableProcessing();
366 SCOPED_TRACE("avoid_unnecessary_blocking_io = true");
371 // This test is to reproduce a bug that read invalid ReadOption in iterator
373 TEST_F(DeleteFileTest
, BackgroundPurgeCopyOptions
) {
374 Options options
= CurrentOptions();
375 SetOptions(&options
);
377 options
.create_if_missing
= true;
380 std::string
first("0"), last("999999");
381 CompactRangeOptions compact_options
;
382 compact_options
.change_level
= true;
383 compact_options
.target_level
= 2;
384 Slice
first_slice(first
), last_slice(last
);
386 // We keep an iterator alive
387 Iterator
* itr
= nullptr;
390 ReadOptions read_options
;
391 read_options
.background_purge_on_iterator_cleanup
= true;
392 itr
= db_
->NewIterator(read_options
);
393 ASSERT_OK(itr
->status());
394 // ReadOptions is deleted, but iterator cleanup function should not be
398 ASSERT_OK(db_
->CompactRange(compact_options
, &first_slice
, &last_slice
));
399 // 3 sst after compaction with live iterator
400 CheckFileTypeCounts(dbname_
, 0, 3, 1);
403 test::SleepingBackgroundTask sleeping_task_after
;
404 env_
->Schedule(&test::SleepingBackgroundTask::DoSleepTask
,
405 &sleeping_task_after
, Env::Priority::HIGH
);
407 // Make sure all background purges are executed
408 sleeping_task_after
.WakeUp();
409 sleeping_task_after
.WaitUntilDone();
410 // 1 sst after iterator deletion
411 CheckFileTypeCounts(dbname_
, 0, 1, 1);
414 TEST_F(DeleteFileTest
, BackgroundPurgeTestMultipleJobs
) {
415 Options options
= CurrentOptions();
416 SetOptions(&options
);
418 options
.create_if_missing
= true;
421 std::string
first("0"), last("999999");
422 CompactRangeOptions compact_options
;
423 compact_options
.change_level
= true;
424 compact_options
.target_level
= 2;
425 Slice
first_slice(first
), last_slice(last
);
427 // We keep an iterator alive
429 ReadOptions read_options
;
430 read_options
.background_purge_on_iterator_cleanup
= true;
431 Iterator
* itr1
= db_
->NewIterator(read_options
);
432 ASSERT_OK(itr1
->status());
434 Iterator
* itr2
= db_
->NewIterator(read_options
);
435 ASSERT_OK(itr2
->status());
436 ASSERT_OK(db_
->CompactRange(compact_options
, &first_slice
, &last_slice
));
437 // 5 sst files after 2 compactions with 2 live iterators
438 CheckFileTypeCounts(dbname_
, 0, 5, 1);
440 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
441 // ~DBImpl should wait until all BGWorkPurge are finished
442 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency(
443 {{"DBImpl::~DBImpl:WaitJob", "DBImpl::BGWorkPurge"},
444 {"DeleteFileTest::GuardFinish",
445 "DeleteFileTest::BackgroundPurgeTestMultipleJobs:DBClose"}});
446 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
449 env_
->Schedule(&DeleteFileTest::DoSleep
, this, Env::Priority::HIGH
);
451 env_
->Schedule(&DeleteFileTest::GuardFinish
, nullptr, Env::Priority::HIGH
);
454 TEST_SYNC_POINT("DeleteFileTest::BackgroundPurgeTestMultipleJobs:DBClose");
455 // 1 sst after iterator deletion
456 CheckFileTypeCounts(dbname_
, 0, 1, 1);
459 TEST_F(DeleteFileTest
, DeleteFileWithIterator
) {
460 Options options
= CurrentOptions();
461 SetOptions(&options
);
463 options
.create_if_missing
= true;
467 ReadOptions read_options
;
468 Iterator
* it
= db_
->NewIterator(read_options
);
469 ASSERT_OK(it
->status());
470 std::vector
<LiveFileMetaData
> metadata
;
471 db_
->GetLiveFilesMetaData(&metadata
);
473 std::string level2file
;
475 ASSERT_EQ(metadata
.size(), static_cast<size_t>(2));
476 if (metadata
[0].level
== 1) {
477 level2file
= metadata
[1].name
;
479 level2file
= metadata
[0].name
;
482 Status status
= db_
->DeleteFile(level2file
);
483 fprintf(stdout
, "Deletion status %s: %s\n", level2file
.c_str(),
484 status
.ToString().c_str());
487 int numKeysIterated
= 0;
488 while (it
->Valid()) {
492 ASSERT_EQ(numKeysIterated
, 50000);
496 TEST_F(DeleteFileTest
, DeleteLogFiles
) {
497 Options options
= CurrentOptions();
498 SetOptions(&options
);
500 options
.create_if_missing
= true;
504 VectorLogPtr logfiles
;
505 ASSERT_OK(db_
->GetSortedWalFiles(logfiles
));
506 ASSERT_GT(logfiles
.size(), 0UL);
507 // Take the last log file which is expected to be alive and try to delete it
508 // Should not succeed because live logs are not allowed to be deleted
509 std::unique_ptr
<LogFile
> alive_log
= std::move(logfiles
.back());
510 ASSERT_EQ(alive_log
->Type(), kAliveLogFile
);
511 ASSERT_OK(env_
->FileExists(wal_dir_
+ "/" + alive_log
->PathName()));
512 fprintf(stdout
, "Deleting alive log file %s\n",
513 alive_log
->PathName().c_str());
514 ASSERT_NOK(db_
->DeleteFile(alive_log
->PathName()));
515 ASSERT_OK(env_
->FileExists(wal_dir_
+ "/" + alive_log
->PathName()));
518 // Call Flush to bring about a new working log file and add more keys
519 // Call Flush again to flush out memtable and move alive log to archived log
520 // and try to delete the archived log file
522 ASSERT_OK(db_
->Flush(fopts
));
524 ASSERT_OK(db_
->Flush(fopts
));
525 ASSERT_OK(db_
->GetSortedWalFiles(logfiles
));
526 ASSERT_GT(logfiles
.size(), 0UL);
527 std::unique_ptr
<LogFile
> archived_log
= std::move(logfiles
.front());
528 ASSERT_EQ(archived_log
->Type(), kArchivedLogFile
);
529 ASSERT_OK(env_
->FileExists(wal_dir_
+ "/" + archived_log
->PathName()));
530 fprintf(stdout
, "Deleting archived log file %s\n",
531 archived_log
->PathName().c_str());
532 ASSERT_OK(db_
->DeleteFile(archived_log
->PathName()));
534 env_
->FileExists(wal_dir_
+ "/" + archived_log
->PathName()).IsNotFound());
537 TEST_F(DeleteFileTest
, DeleteNonDefaultColumnFamily
) {
538 Options options
= CurrentOptions();
539 SetOptions(&options
);
541 options
.create_if_missing
= true;
543 CreateAndReopenWithCF({"new_cf"}, options
);
546 for (int i
= 0; i
< 1000; ++i
) {
547 ASSERT_OK(db_
->Put(WriteOptions(), handles_
[1], test::RandomKey(&rnd
, 10),
548 test::RandomKey(&rnd
, 10)));
550 ASSERT_OK(db_
->Flush(FlushOptions(), handles_
[1]));
551 for (int i
= 0; i
< 1000; ++i
) {
552 ASSERT_OK(db_
->Put(WriteOptions(), handles_
[1], test::RandomKey(&rnd
, 10),
553 test::RandomKey(&rnd
, 10)));
555 ASSERT_OK(db_
->Flush(FlushOptions(), handles_
[1]));
557 std::vector
<LiveFileMetaData
> metadata
;
558 db_
->GetLiveFilesMetaData(&metadata
);
559 ASSERT_EQ(2U, metadata
.size());
560 ASSERT_EQ("new_cf", metadata
[0].column_family_name
);
561 ASSERT_EQ("new_cf", metadata
[1].column_family_name
);
562 auto old_file
= metadata
[0].smallest_seqno
< metadata
[1].smallest_seqno
565 auto new_file
= metadata
[0].smallest_seqno
> metadata
[1].smallest_seqno
568 ASSERT_TRUE(db_
->DeleteFile(new_file
).IsInvalidArgument());
569 ASSERT_OK(db_
->DeleteFile(old_file
));
572 std::unique_ptr
<Iterator
> itr(db_
->NewIterator(ReadOptions(), handles_
[1]));
573 ASSERT_OK(itr
->status());
575 for (itr
->SeekToFirst(); itr
->Valid(); itr
->Next()) {
576 ASSERT_OK(itr
->status());
579 ASSERT_EQ(count
, 1000);
583 ReopenWithColumnFamilies({kDefaultColumnFamilyName
, "new_cf"}, options
);
586 std::unique_ptr
<Iterator
> itr(db_
->NewIterator(ReadOptions(), handles_
[1]));
588 for (itr
->SeekToFirst(); itr
->Valid(); itr
->Next()) {
589 ASSERT_OK(itr
->status());
592 ASSERT_EQ(count
, 1000);
596 } // namespace ROCKSDB_NAMESPACE
598 int main(int argc
, char** argv
) {
599 ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
600 ::testing::InitGoogleTest(&argc
, argv
);
601 RegisterCustomObjects(argc
, argv
);
602 return RUN_ALL_TESTS();
608 int main(int /*argc*/, char** /*argv*/) {
610 "SKIPPED as DBImpl::DeleteFile is not supported in ROCKSDB_LITE\n");
614 #endif // !ROCKSDB_LITE