1 // Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
2 // This source code is licensed under both the GPLv2 (found in the
3 // COPYING file in the root directory) and Apache 2.0 License
4 // (found in the LICENSE.Apache file in the root directory).
6 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
7 // Use of this source code is governed by a BSD-style license that can be
8 // found in the LICENSE file. See the AUTHORS file for names of contributors.
16 #include "db/db_impl.h"
17 #include "db/version_set.h"
18 #include "db/write_batch_internal.h"
19 #include "rocksdb/db.h"
20 #include "rocksdb/env.h"
21 #include "rocksdb/transaction_log.h"
22 #include "util/filename.h"
23 #include "util/string_util.h"
24 #include "util/sync_point.h"
25 #include "util/testharness.h"
26 #include "util/testutil.h"
35 class ObsoleteFilesTest
: public testing::Test
{
45 env_
= Env::Default();
46 // Trigger compaction when the number of level 0 files reaches 2.
47 options_
.level0_file_num_compaction_trigger
= 2;
48 options_
.disable_auto_compactions
= false;
49 options_
.delete_obsolete_files_period_micros
= 0; // always do full purge
50 options_
.enable_thread_tracking
= true;
51 options_
.write_buffer_size
= 1024*1024*1000;
52 options_
.target_file_size_base
= 1024*1024*1000;
53 options_
.max_bytes_for_level_base
= 1024*1024*1000;
54 options_
.WAL_ttl_seconds
= 300; // Used to test log files
55 options_
.WAL_size_limit_MB
= 1024; // Used to test log files
56 dbname_
= test::PerThreadDBPath("obsolete_files_test");
57 options_
.wal_dir
= dbname_
+ "/wal_files";
59 // clean up all the files that might have been there before
60 std::vector
<std::string
> old_files
;
61 env_
->GetChildren(dbname_
, &old_files
);
62 for (auto file
: old_files
) {
63 env_
->DeleteFile(dbname_
+ "/" + file
);
65 env_
->GetChildren(options_
.wal_dir
, &old_files
);
66 for (auto file
: old_files
) {
67 env_
->DeleteFile(options_
.wal_dir
+ "/" + file
);
70 DestroyDB(dbname_
, options_
);
72 EXPECT_OK(ReopenDB(true));
75 Status
ReopenDB(bool create
) {
78 DestroyDB(dbname_
, options_
);
81 options_
.create_if_missing
= create
;
82 return DB::Open(options_
, dbname_
, &db_
);
90 void AddKeys(int numkeys
, int startkey
) {
93 for (int i
= startkey
; i
< (numkeys
+ startkey
) ; i
++) {
94 std::string temp
= ToString(i
);
97 ASSERT_OK(db_
->Put(options
, key
, value
));
102 std::vector
<LiveFileMetaData
> &metadata
,
103 std::vector
<int> *keysperlevel
= nullptr) {
105 if (keysperlevel
!= nullptr) {
106 keysperlevel
->resize(numlevels_
);
110 for (size_t i
= 0; i
< metadata
.size(); i
++) {
111 int startkey
= atoi(metadata
[i
].smallestkey
.c_str());
112 int endkey
= atoi(metadata
[i
].largestkey
.c_str());
113 int numkeysinfile
= (endkey
- startkey
+ 1);
114 numKeys
+= numkeysinfile
;
115 if (keysperlevel
!= nullptr) {
116 (*keysperlevel
)[(int)metadata
[i
].level
] += numkeysinfile
;
118 fprintf(stderr
, "level %d name %s smallest %s largest %s\n",
119 metadata
[i
].level
, metadata
[i
].name
.c_str(),
120 metadata
[i
].smallestkey
.c_str(),
121 metadata
[i
].largestkey
.c_str());
126 void createLevel0Files(int numFiles
, int numKeysPerFile
) {
128 DBImpl
* dbi
= reinterpret_cast<DBImpl
*>(db_
);
129 for (int i
= 0; i
< numFiles
; i
++) {
130 AddKeys(numKeysPerFile
, startKey
);
131 startKey
+= numKeysPerFile
;
132 ASSERT_OK(dbi
->TEST_FlushMemTable());
133 ASSERT_OK(dbi
->TEST_WaitForFlushMemTable());
137 void CheckFileTypeCounts(std::string
& dir
,
140 int required_manifest
) {
141 std::vector
<std::string
> filenames
;
142 env_
->GetChildren(dir
, &filenames
);
144 int log_cnt
= 0, sst_cnt
= 0, manifest_cnt
= 0;
145 for (auto file
: filenames
) {
148 if (ParseFileName(file
, &number
, &type
)) {
149 log_cnt
+= (type
== kLogFile
);
150 sst_cnt
+= (type
== kTableFile
);
151 manifest_cnt
+= (type
== kDescriptorFile
);
154 ASSERT_EQ(required_log
, log_cnt
);
155 ASSERT_EQ(required_sst
, sst_cnt
);
156 ASSERT_EQ(required_manifest
, manifest_cnt
);
160 TEST_F(ObsoleteFilesTest
, RaceForObsoleteFileDeletion
) {
161 SyncPoint::GetInstance()->LoadDependency({
162 {"DBImpl::BackgroundCallCompaction:FoundObsoleteFiles",
163 "ObsoleteFilesTest::RaceForObsoleteFileDeletion:1"},
164 {"DBImpl::BackgroundCallCompaction:PurgedObsoleteFiles",
165 "ObsoleteFilesTest::RaceForObsoleteFileDeletion:2"},
167 SyncPoint::GetInstance()->SetCallBack(
168 "DBImpl::DeleteObsoleteFileImpl:AfterDeletion", [&](void* arg
) {
169 Status
* p_status
= reinterpret_cast<Status
*>(arg
);
170 ASSERT_OK(*p_status
);
172 SyncPoint::GetInstance()->SetCallBack(
173 "DBImpl::CloseHelper:PendingPurgeFinished", [&](void* arg
) {
174 std::vector
<uint64_t>* files_grabbed_for_purge_ptr
=
175 reinterpret_cast<std::vector
<uint64_t>*>(arg
);
176 ASSERT_TRUE(files_grabbed_for_purge_ptr
->empty());
178 SyncPoint::GetInstance()->EnableProcessing();
180 createLevel0Files(2, 50000);
181 CheckFileTypeCounts(options_
.wal_dir
, 1, 0, 0);
183 DBImpl
* dbi
= reinterpret_cast<DBImpl
*>(db_
);
184 port::Thread
user_thread([&]() {
185 JobContext
jobCxt(0);
186 TEST_SYNC_POINT("ObsoleteFilesTest::RaceForObsoleteFileDeletion:1");
187 dbi
->TEST_LockMutex();
188 dbi
->FindObsoleteFiles(&jobCxt
,
189 true /* force=true */, false /* no_full_scan=false */);
190 dbi
->TEST_UnlockMutex();
191 TEST_SYNC_POINT("ObsoleteFilesTest::RaceForObsoleteFileDeletion:2");
192 dbi
->PurgeObsoleteFiles(jobCxt
);
199 SyncPoint::GetInstance()->DisableProcessing();
202 TEST_F(ObsoleteFilesTest
, DeleteObsoleteOptionsFile
) {
203 std::vector
<uint64_t> optsfiles_nums
;
204 std::vector
<bool> optsfiles_keep
;
205 SyncPoint::GetInstance()->SetCallBack(
206 "DBImpl::PurgeObsoleteFiles:CheckOptionsFiles:1", [&](void* arg
) {
207 optsfiles_nums
.push_back(*reinterpret_cast<uint64_t*>(arg
));
209 SyncPoint::GetInstance()->SetCallBack(
210 "DBImpl::PurgeObsoleteFiles:CheckOptionsFiles:2", [&](void* arg
) {
211 optsfiles_keep
.push_back(*reinterpret_cast<bool*>(arg
));
213 SyncPoint::GetInstance()->EnableProcessing();
215 createLevel0Files(2, 50000);
216 CheckFileTypeCounts(options_
.wal_dir
, 1, 0, 0);
218 DBImpl
* dbi
= static_cast<DBImpl
*>(db_
);
219 ASSERT_OK(dbi
->DisableFileDeletions());
220 for (int i
= 0; i
!= 4; ++i
) {
222 ASSERT_OK(dbi
->SetOptions(dbi
->DefaultColumnFamily(),
223 {{"paranoid_file_checks", "false"}}));
225 ASSERT_OK(dbi
->SetOptions(dbi
->DefaultColumnFamily(),
226 {{"paranoid_file_checks", "true"}}));
229 ASSERT_OK(dbi
->EnableFileDeletions(true /* force */));
230 ASSERT_EQ(optsfiles_nums
.size(), optsfiles_keep
.size());
234 std::vector
<std::string
> files
;
235 int opts_file_count
= 0;
236 ASSERT_OK(env_
->GetChildren(dbname_
, &files
));
237 for (const auto& file
: files
) {
239 Slice dummy_info_log_name_prefix
;
241 WalFileType log_type
;
242 if (ParseFileName(file
, &file_num
, dummy_info_log_name_prefix
, &type
,
244 type
== kOptionsFile
) {
248 ASSERT_EQ(2, opts_file_count
);
249 SyncPoint::GetInstance()->DisableProcessing();
252 } //namespace rocksdb
254 int main(int argc
, char** argv
) {
255 ::testing::InitGoogleTest(&argc
, argv
);
256 return RUN_ALL_TESTS();
262 int main(int /*argc*/, char** /*argv*/) {
264 "SKIPPED as DBImpl::DeleteFile is not supported in ROCKSDB_LITE\n");
268 #endif // !ROCKSDB_LITE