1 // Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
2 // This source code is licensed under both the GPLv2 (found in the
3 // COPYING file in the root directory) and Apache 2.0 License
4 // (found in the LICENSE.Apache file in the root directory).
6 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
7 // Use of this source code is governed by a BSD-style license that can be
8 // found in the LICENSE file. See the AUTHORS file for names of contributors.
16 #include "db/db_impl/db_impl.h"
17 #include "db/db_test_util.h"
18 #include "db/version_set.h"
19 #include "db/write_batch_internal.h"
20 #include "file/filename.h"
21 #include "port/stack_trace.h"
22 #include "rocksdb/db.h"
23 #include "rocksdb/env.h"
24 #include "rocksdb/transaction_log.h"
25 #include "test_util/sync_point.h"
26 #include "test_util/testharness.h"
27 #include "test_util/testutil.h"
28 #include "util/string_util.h"
30 namespace ROCKSDB_NAMESPACE
{
32 class DeleteFileTest
: public DBTestBase
{
35 const std::string wal_dir_
;
38 : DBTestBase("/deletefile_test"),
40 wal_dir_(dbname_
+ "/wal_files") {}
42 void SetOptions(Options
* options
) {
44 options
->delete_obsolete_files_period_micros
= 0; // always do full purge
45 options
->enable_thread_tracking
= true;
46 options
->write_buffer_size
= 1024 * 1024 * 1000;
47 options
->target_file_size_base
= 1024 * 1024 * 1000;
48 options
->max_bytes_for_level_base
= 1024 * 1024 * 1000;
49 options
->WAL_ttl_seconds
= 300; // Used to test log files
50 options
->WAL_size_limit_MB
= 1024; // Used to test log files
51 options
->wal_dir
= wal_dir_
;
54 void AddKeys(int numkeys
, int startkey
= 0) {
58 for (int i
= startkey
; i
< (numkeys
+ startkey
) ; i
++) {
59 std::string temp
= ToString(i
);
62 ASSERT_OK(db_
->Put(options
, key
, value
));
67 std::vector
<LiveFileMetaData
> &metadata
,
68 std::vector
<int> *keysperlevel
= nullptr) {
70 if (keysperlevel
!= nullptr) {
71 keysperlevel
->resize(numlevels_
);
75 for (size_t i
= 0; i
< metadata
.size(); i
++) {
76 int startkey
= atoi(metadata
[i
].smallestkey
.c_str());
77 int endkey
= atoi(metadata
[i
].largestkey
.c_str());
78 int numkeysinfile
= (endkey
- startkey
+ 1);
79 numKeys
+= numkeysinfile
;
80 if (keysperlevel
!= nullptr) {
81 (*keysperlevel
)[(int)metadata
[i
].level
] += numkeysinfile
;
83 fprintf(stderr
, "level %d name %s smallest %s largest %s\n",
84 metadata
[i
].level
, metadata
[i
].name
.c_str(),
85 metadata
[i
].smallestkey
.c_str(),
86 metadata
[i
].largestkey
.c_str());
91 void CreateTwoLevels() {
92 AddKeys(50000, 10000);
93 ASSERT_OK(dbfull()->TEST_FlushMemTable());
94 ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
95 for (int i
= 0; i
< 2; ++i
) {
96 ASSERT_OK(dbfull()->TEST_CompactRange(i
, nullptr, nullptr));
99 AddKeys(50000, 10000);
100 ASSERT_OK(dbfull()->TEST_FlushMemTable());
101 ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
102 ASSERT_OK(dbfull()->TEST_CompactRange(0, nullptr, nullptr));
105 void CheckFileTypeCounts(const std::string
& dir
, int required_log
,
106 int required_sst
, int required_manifest
) {
107 std::vector
<std::string
> filenames
;
108 env_
->GetChildren(dir
, &filenames
);
110 int log_cnt
= 0, sst_cnt
= 0, manifest_cnt
= 0;
111 for (auto file
: filenames
) {
114 if (ParseFileName(file
, &number
, &type
)) {
115 log_cnt
+= (type
== kLogFile
);
116 sst_cnt
+= (type
== kTableFile
);
117 manifest_cnt
+= (type
== kDescriptorFile
);
120 ASSERT_EQ(required_log
, log_cnt
);
121 ASSERT_EQ(required_sst
, sst_cnt
);
122 ASSERT_EQ(required_manifest
, manifest_cnt
);
125 static void DoSleep(void* arg
) {
126 auto test
= reinterpret_cast<DeleteFileTest
*>(arg
);
127 test
->env_
->SleepForMicroseconds(2 * 1000 * 1000);
130 // An empty job to guard all jobs are processed
131 static void GuardFinish(void* /*arg*/) {
132 TEST_SYNC_POINT("DeleteFileTest::GuardFinish");
136 TEST_F(DeleteFileTest
, AddKeysAndQueryLevels
) {
137 Options options
= CurrentOptions();
138 SetOptions(&options
);
140 options
.create_if_missing
= true;
144 std::vector
<LiveFileMetaData
> metadata
;
145 db_
->GetLiveFilesMetaData(&metadata
);
147 std::string level1file
= "";
148 int level1keycount
= 0;
149 std::string level2file
= "";
150 int level2keycount
= 0;
154 ASSERT_EQ((int)metadata
.size(), 2);
155 if (metadata
[0].level
== 2) {
160 level1file
= metadata
[level1index
].name
;
161 int startkey
= atoi(metadata
[level1index
].smallestkey
.c_str());
162 int endkey
= atoi(metadata
[level1index
].largestkey
.c_str());
163 level1keycount
= (endkey
- startkey
+ 1);
164 level2file
= metadata
[level2index
].name
;
165 startkey
= atoi(metadata
[level2index
].smallestkey
.c_str());
166 endkey
= atoi(metadata
[level2index
].largestkey
.c_str());
167 level2keycount
= (endkey
- startkey
+ 1);
169 // COntrolled setup. Levels 1 and 2 should both have 50K files.
170 // This is a little fragile as it depends on the current
171 // compaction heuristics.
172 ASSERT_EQ(level1keycount
, 50000);
173 ASSERT_EQ(level2keycount
, 50000);
175 Status status
= db_
->DeleteFile("0.sst");
176 ASSERT_TRUE(status
.IsInvalidArgument());
178 // intermediate level files cannot be deleted.
179 status
= db_
->DeleteFile(level1file
);
180 ASSERT_TRUE(status
.IsInvalidArgument());
182 // Lowest level file deletion should succeed.
183 ASSERT_OK(db_
->DeleteFile(level2file
));
186 TEST_F(DeleteFileTest
, PurgeObsoleteFilesTest
) {
187 Options options
= CurrentOptions();
188 SetOptions(&options
);
190 options
.create_if_missing
= true;
194 // there should be only one (empty) log file because CreateTwoLevels()
195 // flushes the memtables to disk
196 CheckFileTypeCounts(wal_dir_
, 1, 0, 0);
197 // 2 ssts, 1 manifest
198 CheckFileTypeCounts(dbname_
, 0, 2, 1);
199 std::string
first("0"), last("999999");
200 CompactRangeOptions compact_options
;
201 compact_options
.change_level
= true;
202 compact_options
.target_level
= 2;
203 Slice
first_slice(first
), last_slice(last
);
204 db_
->CompactRange(compact_options
, &first_slice
, &last_slice
);
205 // 1 sst after compaction
206 CheckFileTypeCounts(dbname_
, 0, 1, 1);
208 // this time, we keep an iterator alive
210 Iterator
*itr
= nullptr;
212 itr
= db_
->NewIterator(ReadOptions());
213 db_
->CompactRange(compact_options
, &first_slice
, &last_slice
);
214 // 3 sst after compaction with live iterator
215 CheckFileTypeCounts(dbname_
, 0, 3, 1);
217 // 1 sst after iterator deletion
218 CheckFileTypeCounts(dbname_
, 0, 1, 1);
221 TEST_F(DeleteFileTest
, BackgroundPurgeIteratorTest
) {
222 Options options
= CurrentOptions();
223 SetOptions(&options
);
225 options
.create_if_missing
= true;
228 std::string
first("0"), last("999999");
229 CompactRangeOptions compact_options
;
230 compact_options
.change_level
= true;
231 compact_options
.target_level
= 2;
232 Slice
first_slice(first
), last_slice(last
);
234 // We keep an iterator alive
235 Iterator
* itr
= nullptr;
237 ReadOptions read_options
;
238 read_options
.background_purge_on_iterator_cleanup
= true;
239 itr
= db_
->NewIterator(read_options
);
240 db_
->CompactRange(compact_options
, &first_slice
, &last_slice
);
241 // 3 sst after compaction with live iterator
242 CheckFileTypeCounts(dbname_
, 0, 3, 1);
243 test::SleepingBackgroundTask sleeping_task_before
;
244 env_
->Schedule(&test::SleepingBackgroundTask::DoSleepTask
,
245 &sleeping_task_before
, Env::Priority::HIGH
);
247 test::SleepingBackgroundTask sleeping_task_after
;
248 env_
->Schedule(&test::SleepingBackgroundTask::DoSleepTask
,
249 &sleeping_task_after
, Env::Priority::HIGH
);
251 // Make sure no purges are executed foreground
252 CheckFileTypeCounts(dbname_
, 0, 3, 1);
253 sleeping_task_before
.WakeUp();
254 sleeping_task_before
.WaitUntilDone();
256 // Make sure all background purges are executed
257 sleeping_task_after
.WakeUp();
258 sleeping_task_after
.WaitUntilDone();
259 // 1 sst after iterator deletion
260 CheckFileTypeCounts(dbname_
, 0, 1, 1);
263 TEST_F(DeleteFileTest
, BackgroundPurgeCFDropTest
) {
264 Options options
= CurrentOptions();
265 SetOptions(&options
);
267 options
.create_if_missing
= true;
270 auto do_test
= [&](bool bg_purge
) {
271 ColumnFamilyOptions co
;
272 co
.max_write_buffer_size_to_maintain
=
273 static_cast<int64_t>(co
.write_buffer_size
);
276 ColumnFamilyHandle
* cfh
= nullptr;
278 ASSERT_OK(db_
->CreateColumnFamily(co
, "dropme", &cfh
));
280 ASSERT_OK(db_
->Put(wo
, cfh
, "pika", "chu"));
281 ASSERT_OK(db_
->Flush(fo
, cfh
));
282 // Expect 1 sst file.
283 CheckFileTypeCounts(dbname_
, 0, 1, 1);
285 ASSERT_OK(db_
->DropColumnFamily(cfh
));
286 // Still 1 file, it won't be deleted while ColumnFamilyHandle is alive.
287 CheckFileTypeCounts(dbname_
, 0, 1, 1);
290 test::SleepingBackgroundTask sleeping_task_after
;
291 env_
->Schedule(&test::SleepingBackgroundTask::DoSleepTask
,
292 &sleeping_task_after
, Env::Priority::HIGH
);
293 // If background purge is enabled, the file should still be there.
294 CheckFileTypeCounts(dbname_
, 0, bg_purge
? 1 : 0, 1);
295 TEST_SYNC_POINT("DeleteFileTest::BackgroundPurgeCFDropTest:1");
297 // Execute background purges.
298 sleeping_task_after
.WakeUp();
299 sleeping_task_after
.WaitUntilDone();
300 // The file should have been deleted.
301 CheckFileTypeCounts(dbname_
, 0, 0, 1);
305 SCOPED_TRACE("avoid_unnecessary_blocking_io = false");
309 SyncPoint::GetInstance()->DisableProcessing();
310 SyncPoint::GetInstance()->ClearAllCallBacks();
311 SyncPoint::GetInstance()->LoadDependency(
312 {{"DeleteFileTest::BackgroundPurgeCFDropTest:1",
313 "DBImpl::BGWorkPurge:start"}});
314 SyncPoint::GetInstance()->EnableProcessing();
316 options
.avoid_unnecessary_blocking_io
= true;
317 options
.create_if_missing
= false;
320 SCOPED_TRACE("avoid_unnecessary_blocking_io = true");
325 // This test is to reproduce a bug that read invalid ReadOption in iterator
327 TEST_F(DeleteFileTest
, BackgroundPurgeCopyOptions
) {
328 Options options
= CurrentOptions();
329 SetOptions(&options
);
331 options
.create_if_missing
= true;
334 std::string
first("0"), last("999999");
335 CompactRangeOptions compact_options
;
336 compact_options
.change_level
= true;
337 compact_options
.target_level
= 2;
338 Slice
first_slice(first
), last_slice(last
);
340 // We keep an iterator alive
341 Iterator
* itr
= nullptr;
344 ReadOptions read_options
;
345 read_options
.background_purge_on_iterator_cleanup
= true;
346 itr
= db_
->NewIterator(read_options
);
347 // ReadOptions is deleted, but iterator cleanup function should not be
351 db_
->CompactRange(compact_options
, &first_slice
, &last_slice
);
352 // 3 sst after compaction with live iterator
353 CheckFileTypeCounts(dbname_
, 0, 3, 1);
356 test::SleepingBackgroundTask sleeping_task_after
;
357 env_
->Schedule(&test::SleepingBackgroundTask::DoSleepTask
,
358 &sleeping_task_after
, Env::Priority::HIGH
);
360 // Make sure all background purges are executed
361 sleeping_task_after
.WakeUp();
362 sleeping_task_after
.WaitUntilDone();
363 // 1 sst after iterator deletion
364 CheckFileTypeCounts(dbname_
, 0, 1, 1);
367 TEST_F(DeleteFileTest
, BackgroundPurgeTestMultipleJobs
) {
368 Options options
= CurrentOptions();
369 SetOptions(&options
);
371 options
.create_if_missing
= true;
374 std::string
first("0"), last("999999");
375 CompactRangeOptions compact_options
;
376 compact_options
.change_level
= true;
377 compact_options
.target_level
= 2;
378 Slice
first_slice(first
), last_slice(last
);
380 // We keep an iterator alive
382 ReadOptions read_options
;
383 read_options
.background_purge_on_iterator_cleanup
= true;
384 Iterator
* itr1
= db_
->NewIterator(read_options
);
386 Iterator
* itr2
= db_
->NewIterator(read_options
);
387 db_
->CompactRange(compact_options
, &first_slice
, &last_slice
);
388 // 5 sst files after 2 compactions with 2 live iterators
389 CheckFileTypeCounts(dbname_
, 0, 5, 1);
391 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
392 // ~DBImpl should wait until all BGWorkPurge are finished
393 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency(
394 {{"DBImpl::~DBImpl:WaitJob", "DBImpl::BGWorkPurge"},
395 {"DeleteFileTest::GuardFinish",
396 "DeleteFileTest::BackgroundPurgeTestMultipleJobs:DBClose"}});
397 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
400 env_
->Schedule(&DeleteFileTest::DoSleep
, this, Env::Priority::HIGH
);
402 env_
->Schedule(&DeleteFileTest::GuardFinish
, nullptr, Env::Priority::HIGH
);
405 TEST_SYNC_POINT("DeleteFileTest::BackgroundPurgeTestMultipleJobs:DBClose");
406 // 1 sst after iterator deletion
407 CheckFileTypeCounts(dbname_
, 0, 1, 1);
410 TEST_F(DeleteFileTest
, DeleteFileWithIterator
) {
411 Options options
= CurrentOptions();
412 SetOptions(&options
);
414 options
.create_if_missing
= true;
418 ReadOptions read_options
;
419 Iterator
* it
= db_
->NewIterator(read_options
);
420 std::vector
<LiveFileMetaData
> metadata
;
421 db_
->GetLiveFilesMetaData(&metadata
);
423 std::string level2file
;
425 ASSERT_EQ(metadata
.size(), static_cast<size_t>(2));
426 if (metadata
[0].level
== 1) {
427 level2file
= metadata
[1].name
;
429 level2file
= metadata
[0].name
;
432 Status status
= db_
->DeleteFile(level2file
);
433 fprintf(stdout
, "Deletion status %s: %s\n",
434 level2file
.c_str(), status
.ToString().c_str());
435 ASSERT_TRUE(status
.ok());
437 int numKeysIterated
= 0;
442 ASSERT_EQ(numKeysIterated
, 50000);
446 TEST_F(DeleteFileTest
, DeleteLogFiles
) {
447 Options options
= CurrentOptions();
448 SetOptions(&options
);
450 options
.create_if_missing
= true;
454 VectorLogPtr logfiles
;
455 db_
->GetSortedWalFiles(logfiles
);
456 ASSERT_GT(logfiles
.size(), 0UL);
457 // Take the last log file which is expected to be alive and try to delete it
458 // Should not succeed because live logs are not allowed to be deleted
459 std::unique_ptr
<LogFile
> alive_log
= std::move(logfiles
.back());
460 ASSERT_EQ(alive_log
->Type(), kAliveLogFile
);
461 ASSERT_OK(env_
->FileExists(wal_dir_
+ "/" + alive_log
->PathName()));
462 fprintf(stdout
, "Deleting alive log file %s\n",
463 alive_log
->PathName().c_str());
464 ASSERT_TRUE(!db_
->DeleteFile(alive_log
->PathName()).ok());
465 ASSERT_OK(env_
->FileExists(wal_dir_
+ "/" + alive_log
->PathName()));
468 // Call Flush to bring about a new working log file and add more keys
469 // Call Flush again to flush out memtable and move alive log to archived log
470 // and try to delete the archived log file
475 db_
->GetSortedWalFiles(logfiles
);
476 ASSERT_GT(logfiles
.size(), 0UL);
477 std::unique_ptr
<LogFile
> archived_log
= std::move(logfiles
.front());
478 ASSERT_EQ(archived_log
->Type(), kArchivedLogFile
);
479 ASSERT_OK(env_
->FileExists(wal_dir_
+ "/" + archived_log
->PathName()));
480 fprintf(stdout
, "Deleting archived log file %s\n",
481 archived_log
->PathName().c_str());
482 ASSERT_OK(db_
->DeleteFile(archived_log
->PathName()));
483 ASSERT_EQ(Status::NotFound(),
484 env_
->FileExists(wal_dir_
+ "/" + archived_log
->PathName()));
487 TEST_F(DeleteFileTest
, DeleteNonDefaultColumnFamily
) {
488 Options options
= CurrentOptions();
489 SetOptions(&options
);
491 options
.create_if_missing
= true;
493 CreateAndReopenWithCF({"new_cf"}, options
);
496 for (int i
= 0; i
< 1000; ++i
) {
497 ASSERT_OK(db_
->Put(WriteOptions(), handles_
[1], test::RandomKey(&rnd
, 10),
498 test::RandomKey(&rnd
, 10)));
500 ASSERT_OK(db_
->Flush(FlushOptions(), handles_
[1]));
501 for (int i
= 0; i
< 1000; ++i
) {
502 ASSERT_OK(db_
->Put(WriteOptions(), handles_
[1], test::RandomKey(&rnd
, 10),
503 test::RandomKey(&rnd
, 10)));
505 ASSERT_OK(db_
->Flush(FlushOptions(), handles_
[1]));
507 std::vector
<LiveFileMetaData
> metadata
;
508 db_
->GetLiveFilesMetaData(&metadata
);
509 ASSERT_EQ(2U, metadata
.size());
510 ASSERT_EQ("new_cf", metadata
[0].column_family_name
);
511 ASSERT_EQ("new_cf", metadata
[1].column_family_name
);
512 auto old_file
= metadata
[0].smallest_seqno
< metadata
[1].smallest_seqno
515 auto new_file
= metadata
[0].smallest_seqno
> metadata
[1].smallest_seqno
518 ASSERT_TRUE(db_
->DeleteFile(new_file
).IsInvalidArgument());
519 ASSERT_OK(db_
->DeleteFile(old_file
));
522 std::unique_ptr
<Iterator
> itr(db_
->NewIterator(ReadOptions(), handles_
[1]));
524 for (itr
->SeekToFirst(); itr
->Valid(); itr
->Next()) {
525 ASSERT_OK(itr
->status());
528 ASSERT_EQ(count
, 1000);
532 ReopenWithColumnFamilies({kDefaultColumnFamilyName
, "new_cf"}, options
);
535 std::unique_ptr
<Iterator
> itr(db_
->NewIterator(ReadOptions(), handles_
[1]));
537 for (itr
->SeekToFirst(); itr
->Valid(); itr
->Next()) {
538 ASSERT_OK(itr
->status());
541 ASSERT_EQ(count
, 1000);
545 } // namespace ROCKSDB_NAMESPACE
547 #ifdef ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS
549 void RegisterCustomObjects(int argc
, char** argv
);
552 void RegisterCustomObjects(int /*argc*/, char** /*argv*/) {}
553 #endif // !ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS
555 int main(int argc
, char** argv
) {
556 ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
557 ::testing::InitGoogleTest(&argc
, argv
);
558 RegisterCustomObjects(argc
, argv
);
559 return RUN_ALL_TESTS();
565 int main(int /*argc*/, char** /*argv*/) {
567 "SKIPPED as DBImpl::DeleteFile is not supported in ROCKSDB_LITE\n");
571 #endif // !ROCKSDB_LITE