1 // Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
2 // This source code is licensed under both the GPLv2 (found in the
3 // COPYING file in the root directory) and Apache 2.0 License
4 // (found in the LICENSE.Apache file in the root directory).
6 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
7 // Use of this source code is governed by a BSD-style license that can be
8 // found in the LICENSE file. See the AUTHORS file for names of contributors.
16 #include "db/db_impl.h"
17 #include "db/version_set.h"
18 #include "db/write_batch_internal.h"
19 #include "rocksdb/db.h"
20 #include "rocksdb/env.h"
21 #include "rocksdb/transaction_log.h"
22 #include "util/filename.h"
23 #include "util/string_util.h"
24 #include "util/sync_point.h"
25 #include "util/testharness.h"
26 #include "util/testutil.h"
30 class DeleteFileTest
: public testing::Test
{
40 env_
= Env::Default();
41 options_
.delete_obsolete_files_period_micros
= 0; // always do full purge
42 options_
.enable_thread_tracking
= true;
43 options_
.write_buffer_size
= 1024*1024*1000;
44 options_
.target_file_size_base
= 1024*1024*1000;
45 options_
.max_bytes_for_level_base
= 1024*1024*1000;
46 options_
.WAL_ttl_seconds
= 300; // Used to test log files
47 options_
.WAL_size_limit_MB
= 1024; // Used to test log files
48 dbname_
= test::PerThreadDBPath("deletefile_test");
49 options_
.wal_dir
= dbname_
+ "/wal_files";
51 // clean up all the files that might have been there before
52 std::vector
<std::string
> old_files
;
53 env_
->GetChildren(dbname_
, &old_files
);
54 for (auto file
: old_files
) {
55 env_
->DeleteFile(dbname_
+ "/" + file
);
57 env_
->GetChildren(options_
.wal_dir
, &old_files
);
58 for (auto file
: old_files
) {
59 env_
->DeleteFile(options_
.wal_dir
+ "/" + file
);
62 DestroyDB(dbname_
, options_
);
64 EXPECT_OK(ReopenDB(true));
67 Status
ReopenDB(bool create
) {
70 DestroyDB(dbname_
, options_
);
73 options_
.create_if_missing
= create
;
74 Status s
= DB::Open(options_
, dbname_
, &db_
);
84 void AddKeys(int numkeys
, int startkey
= 0) {
88 for (int i
= startkey
; i
< (numkeys
+ startkey
) ; i
++) {
89 std::string temp
= ToString(i
);
92 ASSERT_OK(db_
->Put(options
, key
, value
));
97 std::vector
<LiveFileMetaData
> &metadata
,
98 std::vector
<int> *keysperlevel
= nullptr) {
100 if (keysperlevel
!= nullptr) {
101 keysperlevel
->resize(numlevels_
);
105 for (size_t i
= 0; i
< metadata
.size(); i
++) {
106 int startkey
= atoi(metadata
[i
].smallestkey
.c_str());
107 int endkey
= atoi(metadata
[i
].largestkey
.c_str());
108 int numkeysinfile
= (endkey
- startkey
+ 1);
109 numKeys
+= numkeysinfile
;
110 if (keysperlevel
!= nullptr) {
111 (*keysperlevel
)[(int)metadata
[i
].level
] += numkeysinfile
;
113 fprintf(stderr
, "level %d name %s smallest %s largest %s\n",
114 metadata
[i
].level
, metadata
[i
].name
.c_str(),
115 metadata
[i
].smallestkey
.c_str(),
116 metadata
[i
].largestkey
.c_str());
121 void CreateTwoLevels() {
122 AddKeys(50000, 10000);
123 DBImpl
* dbi
= reinterpret_cast<DBImpl
*>(db_
);
124 ASSERT_OK(dbi
->TEST_FlushMemTable());
125 ASSERT_OK(dbi
->TEST_WaitForFlushMemTable());
126 for (int i
= 0; i
< 2; ++i
) {
127 ASSERT_OK(dbi
->TEST_CompactRange(i
, nullptr, nullptr));
130 AddKeys(50000, 10000);
131 ASSERT_OK(dbi
->TEST_FlushMemTable());
132 ASSERT_OK(dbi
->TEST_WaitForFlushMemTable());
133 ASSERT_OK(dbi
->TEST_CompactRange(0, nullptr, nullptr));
136 void CheckFileTypeCounts(std::string
& dir
,
139 int required_manifest
) {
140 std::vector
<std::string
> filenames
;
141 env_
->GetChildren(dir
, &filenames
);
143 int log_cnt
= 0, sst_cnt
= 0, manifest_cnt
= 0;
144 for (auto file
: filenames
) {
147 if (ParseFileName(file
, &number
, &type
)) {
148 log_cnt
+= (type
== kLogFile
);
149 sst_cnt
+= (type
== kTableFile
);
150 manifest_cnt
+= (type
== kDescriptorFile
);
153 ASSERT_EQ(required_log
, log_cnt
);
154 ASSERT_EQ(required_sst
, sst_cnt
);
155 ASSERT_EQ(required_manifest
, manifest_cnt
);
158 static void DoSleep(void* arg
) {
159 auto test
= reinterpret_cast<DeleteFileTest
*>(arg
);
160 test
->env_
->SleepForMicroseconds(2 * 1000 * 1000);
163 // An empty job to guard all jobs are processed
164 static void GuardFinish(void* /*arg*/) {
165 TEST_SYNC_POINT("DeleteFileTest::GuardFinish");
169 TEST_F(DeleteFileTest
, AddKeysAndQueryLevels
) {
171 std::vector
<LiveFileMetaData
> metadata
;
172 db_
->GetLiveFilesMetaData(&metadata
);
174 std::string level1file
= "";
175 int level1keycount
= 0;
176 std::string level2file
= "";
177 int level2keycount
= 0;
181 ASSERT_EQ((int)metadata
.size(), 2);
182 if (metadata
[0].level
== 2) {
187 level1file
= metadata
[level1index
].name
;
188 int startkey
= atoi(metadata
[level1index
].smallestkey
.c_str());
189 int endkey
= atoi(metadata
[level1index
].largestkey
.c_str());
190 level1keycount
= (endkey
- startkey
+ 1);
191 level2file
= metadata
[level2index
].name
;
192 startkey
= atoi(metadata
[level2index
].smallestkey
.c_str());
193 endkey
= atoi(metadata
[level2index
].largestkey
.c_str());
194 level2keycount
= (endkey
- startkey
+ 1);
196 // COntrolled setup. Levels 1 and 2 should both have 50K files.
197 // This is a little fragile as it depends on the current
198 // compaction heuristics.
199 ASSERT_EQ(level1keycount
, 50000);
200 ASSERT_EQ(level2keycount
, 50000);
202 Status status
= db_
->DeleteFile("0.sst");
203 ASSERT_TRUE(status
.IsInvalidArgument());
205 // intermediate level files cannot be deleted.
206 status
= db_
->DeleteFile(level1file
);
207 ASSERT_TRUE(status
.IsInvalidArgument());
209 // Lowest level file deletion should succeed.
210 ASSERT_OK(db_
->DeleteFile(level2file
));
215 TEST_F(DeleteFileTest
, PurgeObsoleteFilesTest
) {
217 // there should be only one (empty) log file because CreateTwoLevels()
218 // flushes the memtables to disk
219 CheckFileTypeCounts(options_
.wal_dir
, 1, 0, 0);
220 // 2 ssts, 1 manifest
221 CheckFileTypeCounts(dbname_
, 0, 2, 1);
222 std::string
first("0"), last("999999");
223 CompactRangeOptions compact_options
;
224 compact_options
.change_level
= true;
225 compact_options
.target_level
= 2;
226 Slice
first_slice(first
), last_slice(last
);
227 db_
->CompactRange(compact_options
, &first_slice
, &last_slice
);
228 // 1 sst after compaction
229 CheckFileTypeCounts(dbname_
, 0, 1, 1);
231 // this time, we keep an iterator alive
233 Iterator
*itr
= nullptr;
235 itr
= db_
->NewIterator(ReadOptions());
236 db_
->CompactRange(compact_options
, &first_slice
, &last_slice
);
237 // 3 sst after compaction with live iterator
238 CheckFileTypeCounts(dbname_
, 0, 3, 1);
240 // 1 sst after iterator deletion
241 CheckFileTypeCounts(dbname_
, 0, 1, 1);
246 TEST_F(DeleteFileTest
, BackgroundPurgeIteratorTest
) {
247 std::string
first("0"), last("999999");
248 CompactRangeOptions compact_options
;
249 compact_options
.change_level
= true;
250 compact_options
.target_level
= 2;
251 Slice
first_slice(first
), last_slice(last
);
253 // We keep an iterator alive
254 Iterator
* itr
= nullptr;
257 options
.background_purge_on_iterator_cleanup
= true;
258 itr
= db_
->NewIterator(options
);
259 db_
->CompactRange(compact_options
, &first_slice
, &last_slice
);
260 // 3 sst after compaction with live iterator
261 CheckFileTypeCounts(dbname_
, 0, 3, 1);
262 test::SleepingBackgroundTask sleeping_task_before
;
263 env_
->Schedule(&test::SleepingBackgroundTask::DoSleepTask
,
264 &sleeping_task_before
, Env::Priority::HIGH
);
266 test::SleepingBackgroundTask sleeping_task_after
;
267 env_
->Schedule(&test::SleepingBackgroundTask::DoSleepTask
,
268 &sleeping_task_after
, Env::Priority::HIGH
);
270 // Make sure no purges are executed foreground
271 CheckFileTypeCounts(dbname_
, 0, 3, 1);
272 sleeping_task_before
.WakeUp();
273 sleeping_task_before
.WaitUntilDone();
275 // Make sure all background purges are executed
276 sleeping_task_after
.WakeUp();
277 sleeping_task_after
.WaitUntilDone();
278 // 1 sst after iterator deletion
279 CheckFileTypeCounts(dbname_
, 0, 1, 1);
284 TEST_F(DeleteFileTest
, BackgroundPurgeCFDropTest
) {
285 auto do_test
= [&](bool bg_purge
) {
286 ColumnFamilyOptions co
;
289 ColumnFamilyHandle
* cfh
= nullptr;
291 ASSERT_OK(db_
->CreateColumnFamily(co
, "dropme", &cfh
));
293 ASSERT_OK(db_
->Put(wo
, cfh
, "pika", "chu"));
294 ASSERT_OK(db_
->Flush(fo
, cfh
));
295 // Expect 1 sst file.
296 CheckFileTypeCounts(dbname_
, 0, 1, 1);
298 ASSERT_OK(db_
->DropColumnFamily(cfh
));
299 // Still 1 file, it won't be deleted while ColumnFamilyHandle is alive.
300 CheckFileTypeCounts(dbname_
, 0, 1, 1);
303 test::SleepingBackgroundTask sleeping_task_after
;
304 env_
->Schedule(&test::SleepingBackgroundTask::DoSleepTask
,
305 &sleeping_task_after
, Env::Priority::HIGH
);
306 // If background purge is enabled, the file should still be there.
307 CheckFileTypeCounts(dbname_
, 0, bg_purge
? 1 : 0, 1);
309 // Execute background purges.
310 sleeping_task_after
.WakeUp();
311 sleeping_task_after
.WaitUntilDone();
312 // The file should have been deleted.
313 CheckFileTypeCounts(dbname_
, 0, 0, 1);
317 SCOPED_TRACE("avoid_unnecessary_blocking_io = false");
321 options_
.avoid_unnecessary_blocking_io
= true;
322 ASSERT_OK(ReopenDB(false));
324 SCOPED_TRACE("avoid_unnecessary_blocking_io = true");
331 // This test is to reproduce a bug that read invalid ReadOption in iterator
333 TEST_F(DeleteFileTest
, BackgroundPurgeCopyOptions
) {
334 std::string
first("0"), last("999999");
335 CompactRangeOptions compact_options
;
336 compact_options
.change_level
= true;
337 compact_options
.target_level
= 2;
338 Slice
first_slice(first
), last_slice(last
);
340 // We keep an iterator alive
341 Iterator
* itr
= nullptr;
343 ReadOptions
* options
= new ReadOptions();
344 options
->background_purge_on_iterator_cleanup
= true;
345 itr
= db_
->NewIterator(*options
);
346 // ReadOptions is deleted, but iterator cleanup function should not be
350 db_
->CompactRange(compact_options
, &first_slice
, &last_slice
);
351 // 3 sst after compaction with live iterator
352 CheckFileTypeCounts(dbname_
, 0, 3, 1);
355 test::SleepingBackgroundTask sleeping_task_after
;
356 env_
->Schedule(&test::SleepingBackgroundTask::DoSleepTask
,
357 &sleeping_task_after
, Env::Priority::HIGH
);
359 // Make sure all background purges are executed
360 sleeping_task_after
.WakeUp();
361 sleeping_task_after
.WaitUntilDone();
362 // 1 sst after iterator deletion
363 CheckFileTypeCounts(dbname_
, 0, 1, 1);
368 TEST_F(DeleteFileTest
, BackgroundPurgeTestMultipleJobs
) {
369 std::string
first("0"), last("999999");
370 CompactRangeOptions compact_options
;
371 compact_options
.change_level
= true;
372 compact_options
.target_level
= 2;
373 Slice
first_slice(first
), last_slice(last
);
375 // We keep an iterator alive
378 options
.background_purge_on_iterator_cleanup
= true;
379 Iterator
* itr1
= db_
->NewIterator(options
);
381 Iterator
* itr2
= db_
->NewIterator(options
);
382 db_
->CompactRange(compact_options
, &first_slice
, &last_slice
);
383 // 5 sst files after 2 compactions with 2 live iterators
384 CheckFileTypeCounts(dbname_
, 0, 5, 1);
386 // ~DBImpl should wait until all BGWorkPurge are finished
387 rocksdb::SyncPoint::GetInstance()->LoadDependency(
388 {{"DBImpl::~DBImpl:WaitJob", "DBImpl::BGWorkPurge"},
389 {"DeleteFileTest::GuardFinish",
390 "DeleteFileTest::BackgroundPurgeTestMultipleJobs:DBClose"}});
391 rocksdb::SyncPoint::GetInstance()->EnableProcessing();
394 env_
->Schedule(&DeleteFileTest::DoSleep
, this, Env::Priority::HIGH
);
396 env_
->Schedule(&DeleteFileTest::GuardFinish
, nullptr, Env::Priority::HIGH
);
399 TEST_SYNC_POINT("DeleteFileTest::BackgroundPurgeTestMultipleJobs:DBClose");
400 // 1 sst after iterator deletion
401 CheckFileTypeCounts(dbname_
, 0, 1, 1);
402 rocksdb::SyncPoint::GetInstance()->DisableProcessing();
405 TEST_F(DeleteFileTest
, DeleteFileWithIterator
) {
408 Iterator
* it
= db_
->NewIterator(options
);
409 std::vector
<LiveFileMetaData
> metadata
;
410 db_
->GetLiveFilesMetaData(&metadata
);
412 std::string level2file
= "";
414 ASSERT_EQ((int)metadata
.size(), 2);
415 if (metadata
[0].level
== 1) {
416 level2file
= metadata
[1].name
;
418 level2file
= metadata
[0].name
;
421 Status status
= db_
->DeleteFile(level2file
);
422 fprintf(stdout
, "Deletion status %s: %s\n",
423 level2file
.c_str(), status
.ToString().c_str());
424 ASSERT_TRUE(status
.ok());
426 int numKeysIterated
= 0;
431 ASSERT_EQ(numKeysIterated
, 50000);
436 TEST_F(DeleteFileTest
, DeleteLogFiles
) {
438 VectorLogPtr logfiles
;
439 db_
->GetSortedWalFiles(logfiles
);
440 ASSERT_GT(logfiles
.size(), 0UL);
441 // Take the last log file which is expected to be alive and try to delete it
442 // Should not succeed because live logs are not allowed to be deleted
443 std::unique_ptr
<LogFile
> alive_log
= std::move(logfiles
.back());
444 ASSERT_EQ(alive_log
->Type(), kAliveLogFile
);
445 ASSERT_OK(env_
->FileExists(options_
.wal_dir
+ "/" + alive_log
->PathName()));
446 fprintf(stdout
, "Deleting alive log file %s\n",
447 alive_log
->PathName().c_str());
448 ASSERT_TRUE(!db_
->DeleteFile(alive_log
->PathName()).ok());
449 ASSERT_OK(env_
->FileExists(options_
.wal_dir
+ "/" + alive_log
->PathName()));
452 // Call Flush to bring about a new working log file and add more keys
453 // Call Flush again to flush out memtable and move alive log to archived log
454 // and try to delete the archived log file
459 db_
->GetSortedWalFiles(logfiles
);
460 ASSERT_GT(logfiles
.size(), 0UL);
461 std::unique_ptr
<LogFile
> archived_log
= std::move(logfiles
.front());
462 ASSERT_EQ(archived_log
->Type(), kArchivedLogFile
);
464 env_
->FileExists(options_
.wal_dir
+ "/" + archived_log
->PathName()));
465 fprintf(stdout
, "Deleting archived log file %s\n",
466 archived_log
->PathName().c_str());
467 ASSERT_OK(db_
->DeleteFile(archived_log
->PathName()));
468 ASSERT_EQ(Status::NotFound(), env_
->FileExists(options_
.wal_dir
+ "/" +
469 archived_log
->PathName()));
473 TEST_F(DeleteFileTest
, DeleteNonDefaultColumnFamily
) {
475 DBOptions db_options
;
476 db_options
.create_if_missing
= true;
477 db_options
.create_missing_column_families
= true;
478 std::vector
<ColumnFamilyDescriptor
> column_families
;
479 column_families
.emplace_back();
480 column_families
.emplace_back("new_cf", ColumnFamilyOptions());
482 std::vector
<rocksdb::ColumnFamilyHandle
*> handles
;
484 ASSERT_OK(DB::Open(db_options
, dbname_
, column_families
, &handles
, &db
));
487 for (int i
= 0; i
< 1000; ++i
) {
488 ASSERT_OK(db
->Put(WriteOptions(), handles
[1], test::RandomKey(&rnd
, 10),
489 test::RandomKey(&rnd
, 10)));
491 ASSERT_OK(db
->Flush(FlushOptions(), handles
[1]));
492 for (int i
= 0; i
< 1000; ++i
) {
493 ASSERT_OK(db
->Put(WriteOptions(), handles
[1], test::RandomKey(&rnd
, 10),
494 test::RandomKey(&rnd
, 10)));
496 ASSERT_OK(db
->Flush(FlushOptions(), handles
[1]));
498 std::vector
<LiveFileMetaData
> metadata
;
499 db
->GetLiveFilesMetaData(&metadata
);
500 ASSERT_EQ(2U, metadata
.size());
501 ASSERT_EQ("new_cf", metadata
[0].column_family_name
);
502 ASSERT_EQ("new_cf", metadata
[1].column_family_name
);
503 auto old_file
= metadata
[0].smallest_seqno
< metadata
[1].smallest_seqno
506 auto new_file
= metadata
[0].smallest_seqno
> metadata
[1].smallest_seqno
509 ASSERT_TRUE(db
->DeleteFile(new_file
).IsInvalidArgument());
510 ASSERT_OK(db
->DeleteFile(old_file
));
513 std::unique_ptr
<Iterator
> itr(db
->NewIterator(ReadOptions(), handles
[1]));
515 for (itr
->SeekToFirst(); itr
->Valid(); itr
->Next()) {
516 ASSERT_OK(itr
->status());
519 ASSERT_EQ(count
, 1000);
526 ASSERT_OK(DB::Open(db_options
, dbname_
, column_families
, &handles
, &db
));
528 std::unique_ptr
<Iterator
> itr(db
->NewIterator(ReadOptions(), handles
[1]));
530 for (itr
->SeekToFirst(); itr
->Valid(); itr
->Next()) {
531 ASSERT_OK(itr
->status());
534 ASSERT_EQ(count
, 1000);
542 } //namespace rocksdb
544 int main(int argc
, char** argv
) {
545 ::testing::InitGoogleTest(&argc
, argv
);
546 return RUN_ALL_TESTS();
552 int main(int /*argc*/, char** /*argv*/) {
554 "SKIPPED as DBImpl::DeleteFile is not supported in ROCKSDB_LITE\n");
558 #endif // !ROCKSDB_LITE