]>
Commit | Line | Data |
---|---|---|
11fdf7f2 TL |
1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. |
2 | // This source code is licensed under both the GPLv2 (found in the | |
3 | // COPYING file in the root directory) and Apache 2.0 License | |
4 | // (found in the LICENSE.Apache file in the root directory). | |
5 | // | |
6 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. | |
7 | // Use of this source code is governed by a BSD-style license that can be | |
8 | // found in the LICENSE file. See the AUTHORS file for names of contributors. | |
9 | ||
10 | #ifndef ROCKSDB_LITE | |
11 | ||
12 | #include <stdlib.h> | |
13 | #include <map> | |
14 | #include <string> | |
15 | #include <vector> | |
16 | #include "db/db_impl.h" | |
17 | #include "db/version_set.h" | |
18 | #include "db/write_batch_internal.h" | |
19 | #include "rocksdb/db.h" | |
20 | #include "rocksdb/env.h" | |
21 | #include "rocksdb/transaction_log.h" | |
22 | #include "util/filename.h" | |
23 | #include "util/string_util.h" | |
24 | #include "util/sync_point.h" | |
25 | #include "util/testharness.h" | |
26 | #include "util/testutil.h" | |
27 | ||
28 | using std::cerr; | |
29 | using std::cout; | |
30 | using std::endl; | |
31 | using std::flush; | |
32 | ||
33 | namespace rocksdb { | |
34 | ||
35 | class ObsoleteFilesTest : public testing::Test { | |
36 | public: | |
37 | std::string dbname_; | |
38 | Options options_; | |
39 | DB* db_; | |
40 | Env* env_; | |
41 | int numlevels_; | |
42 | ||
43 | ObsoleteFilesTest() { | |
44 | db_ = nullptr; | |
45 | env_ = Env::Default(); | |
46 | // Trigger compaction when the number of level 0 files reaches 2. | |
47 | options_.level0_file_num_compaction_trigger = 2; | |
48 | options_.disable_auto_compactions = false; | |
49 | options_.delete_obsolete_files_period_micros = 0; // always do full purge | |
50 | options_.enable_thread_tracking = true; | |
51 | options_.write_buffer_size = 1024*1024*1000; | |
52 | options_.target_file_size_base = 1024*1024*1000; | |
53 | options_.max_bytes_for_level_base = 1024*1024*1000; | |
54 | options_.WAL_ttl_seconds = 300; // Used to test log files | |
55 | options_.WAL_size_limit_MB = 1024; // Used to test log files | |
56 | dbname_ = test::PerThreadDBPath("obsolete_files_test"); | |
57 | options_.wal_dir = dbname_ + "/wal_files"; | |
58 | ||
59 | // clean up all the files that might have been there before | |
60 | std::vector<std::string> old_files; | |
61 | env_->GetChildren(dbname_, &old_files); | |
62 | for (auto file : old_files) { | |
63 | env_->DeleteFile(dbname_ + "/" + file); | |
64 | } | |
65 | env_->GetChildren(options_.wal_dir, &old_files); | |
66 | for (auto file : old_files) { | |
67 | env_->DeleteFile(options_.wal_dir + "/" + file); | |
68 | } | |
69 | ||
70 | DestroyDB(dbname_, options_); | |
71 | numlevels_ = 7; | |
72 | EXPECT_OK(ReopenDB(true)); | |
73 | } | |
74 | ||
75 | Status ReopenDB(bool create) { | |
76 | delete db_; | |
77 | if (create) { | |
78 | DestroyDB(dbname_, options_); | |
79 | } | |
80 | db_ = nullptr; | |
81 | options_.create_if_missing = create; | |
82 | return DB::Open(options_, dbname_, &db_); | |
83 | } | |
84 | ||
85 | void CloseDB() { | |
86 | delete db_; | |
87 | db_ = nullptr; | |
88 | } | |
89 | ||
90 | void AddKeys(int numkeys, int startkey) { | |
91 | WriteOptions options; | |
92 | options.sync = false; | |
93 | for (int i = startkey; i < (numkeys + startkey) ; i++) { | |
94 | std::string temp = ToString(i); | |
95 | Slice key(temp); | |
96 | Slice value(temp); | |
97 | ASSERT_OK(db_->Put(options, key, value)); | |
98 | } | |
99 | } | |
100 | ||
101 | int numKeysInLevels( | |
102 | std::vector<LiveFileMetaData> &metadata, | |
103 | std::vector<int> *keysperlevel = nullptr) { | |
104 | ||
105 | if (keysperlevel != nullptr) { | |
106 | keysperlevel->resize(numlevels_); | |
107 | } | |
108 | ||
109 | int numKeys = 0; | |
110 | for (size_t i = 0; i < metadata.size(); i++) { | |
111 | int startkey = atoi(metadata[i].smallestkey.c_str()); | |
112 | int endkey = atoi(metadata[i].largestkey.c_str()); | |
113 | int numkeysinfile = (endkey - startkey + 1); | |
114 | numKeys += numkeysinfile; | |
115 | if (keysperlevel != nullptr) { | |
116 | (*keysperlevel)[(int)metadata[i].level] += numkeysinfile; | |
117 | } | |
118 | fprintf(stderr, "level %d name %s smallest %s largest %s\n", | |
119 | metadata[i].level, metadata[i].name.c_str(), | |
120 | metadata[i].smallestkey.c_str(), | |
121 | metadata[i].largestkey.c_str()); | |
122 | } | |
123 | return numKeys; | |
124 | } | |
125 | ||
126 | void createLevel0Files(int numFiles, int numKeysPerFile) { | |
127 | int startKey = 0; | |
128 | DBImpl* dbi = reinterpret_cast<DBImpl*>(db_); | |
129 | for (int i = 0; i < numFiles; i++) { | |
130 | AddKeys(numKeysPerFile, startKey); | |
131 | startKey += numKeysPerFile; | |
132 | ASSERT_OK(dbi->TEST_FlushMemTable()); | |
133 | ASSERT_OK(dbi->TEST_WaitForFlushMemTable()); | |
134 | } | |
135 | } | |
136 | ||
137 | void CheckFileTypeCounts(std::string& dir, | |
138 | int required_log, | |
139 | int required_sst, | |
140 | int required_manifest) { | |
141 | std::vector<std::string> filenames; | |
142 | env_->GetChildren(dir, &filenames); | |
143 | ||
144 | int log_cnt = 0, sst_cnt = 0, manifest_cnt = 0; | |
145 | for (auto file : filenames) { | |
146 | uint64_t number; | |
147 | FileType type; | |
148 | if (ParseFileName(file, &number, &type)) { | |
149 | log_cnt += (type == kLogFile); | |
150 | sst_cnt += (type == kTableFile); | |
151 | manifest_cnt += (type == kDescriptorFile); | |
152 | } | |
153 | } | |
154 | ASSERT_EQ(required_log, log_cnt); | |
155 | ASSERT_EQ(required_sst, sst_cnt); | |
156 | ASSERT_EQ(required_manifest, manifest_cnt); | |
157 | } | |
158 | }; | |
159 | ||
160 | TEST_F(ObsoleteFilesTest, RaceForObsoleteFileDeletion) { | |
11fdf7f2 TL |
161 | SyncPoint::GetInstance()->LoadDependency({ |
162 | {"DBImpl::BackgroundCallCompaction:FoundObsoleteFiles", | |
163 | "ObsoleteFilesTest::RaceForObsoleteFileDeletion:1"}, | |
164 | {"DBImpl::BackgroundCallCompaction:PurgedObsoleteFiles", | |
165 | "ObsoleteFilesTest::RaceForObsoleteFileDeletion:2"}, | |
166 | }); | |
167 | SyncPoint::GetInstance()->SetCallBack( | |
168 | "DBImpl::DeleteObsoleteFileImpl:AfterDeletion", [&](void* arg) { | |
169 | Status* p_status = reinterpret_cast<Status*>(arg); | |
494da23a | 170 | ASSERT_OK(*p_status); |
11fdf7f2 TL |
171 | }); |
172 | SyncPoint::GetInstance()->SetCallBack( | |
173 | "DBImpl::CloseHelper:PendingPurgeFinished", [&](void* arg) { | |
174 | std::vector<uint64_t>* files_grabbed_for_purge_ptr = | |
175 | reinterpret_cast<std::vector<uint64_t>*>(arg); | |
176 | ASSERT_TRUE(files_grabbed_for_purge_ptr->empty()); | |
177 | }); | |
178 | SyncPoint::GetInstance()->EnableProcessing(); | |
179 | ||
494da23a TL |
180 | createLevel0Files(2, 50000); |
181 | CheckFileTypeCounts(options_.wal_dir, 1, 0, 0); | |
182 | ||
11fdf7f2 TL |
183 | DBImpl* dbi = reinterpret_cast<DBImpl*>(db_); |
184 | port::Thread user_thread([&]() { | |
185 | JobContext jobCxt(0); | |
186 | TEST_SYNC_POINT("ObsoleteFilesTest::RaceForObsoleteFileDeletion:1"); | |
187 | dbi->TEST_LockMutex(); | |
188 | dbi->FindObsoleteFiles(&jobCxt, | |
189 | true /* force=true */, false /* no_full_scan=false */); | |
190 | dbi->TEST_UnlockMutex(); | |
191 | TEST_SYNC_POINT("ObsoleteFilesTest::RaceForObsoleteFileDeletion:2"); | |
192 | dbi->PurgeObsoleteFiles(jobCxt); | |
193 | jobCxt.Clean(); | |
194 | }); | |
195 | ||
196 | user_thread.join(); | |
197 | ||
198 | CloseDB(); | |
494da23a | 199 | SyncPoint::GetInstance()->DisableProcessing(); |
11fdf7f2 TL |
200 | } |
201 | ||
202 | TEST_F(ObsoleteFilesTest, DeleteObsoleteOptionsFile) { | |
11fdf7f2 TL |
203 | std::vector<uint64_t> optsfiles_nums; |
204 | std::vector<bool> optsfiles_keep; | |
205 | SyncPoint::GetInstance()->SetCallBack( | |
206 | "DBImpl::PurgeObsoleteFiles:CheckOptionsFiles:1", [&](void* arg) { | |
207 | optsfiles_nums.push_back(*reinterpret_cast<uint64_t*>(arg)); | |
208 | }); | |
209 | SyncPoint::GetInstance()->SetCallBack( | |
210 | "DBImpl::PurgeObsoleteFiles:CheckOptionsFiles:2", [&](void* arg) { | |
211 | optsfiles_keep.push_back(*reinterpret_cast<bool*>(arg)); | |
212 | }); | |
213 | SyncPoint::GetInstance()->EnableProcessing(); | |
214 | ||
494da23a TL |
215 | createLevel0Files(2, 50000); |
216 | CheckFileTypeCounts(options_.wal_dir, 1, 0, 0); | |
217 | ||
11fdf7f2 TL |
218 | DBImpl* dbi = static_cast<DBImpl*>(db_); |
219 | ASSERT_OK(dbi->DisableFileDeletions()); | |
220 | for (int i = 0; i != 4; ++i) { | |
221 | if (i % 2) { | |
222 | ASSERT_OK(dbi->SetOptions(dbi->DefaultColumnFamily(), | |
223 | {{"paranoid_file_checks", "false"}})); | |
224 | } else { | |
225 | ASSERT_OK(dbi->SetOptions(dbi->DefaultColumnFamily(), | |
226 | {{"paranoid_file_checks", "true"}})); | |
227 | } | |
228 | } | |
229 | ASSERT_OK(dbi->EnableFileDeletions(true /* force */)); | |
230 | ASSERT_EQ(optsfiles_nums.size(), optsfiles_keep.size()); | |
231 | ||
232 | CloseDB(); | |
233 | ||
234 | std::vector<std::string> files; | |
235 | int opts_file_count = 0; | |
236 | ASSERT_OK(env_->GetChildren(dbname_, &files)); | |
237 | for (const auto& file : files) { | |
238 | uint64_t file_num; | |
239 | Slice dummy_info_log_name_prefix; | |
240 | FileType type; | |
241 | WalFileType log_type; | |
242 | if (ParseFileName(file, &file_num, dummy_info_log_name_prefix, &type, | |
243 | &log_type) && | |
244 | type == kOptionsFile) { | |
245 | opts_file_count++; | |
246 | } | |
247 | } | |
248 | ASSERT_EQ(2, opts_file_count); | |
494da23a | 249 | SyncPoint::GetInstance()->DisableProcessing(); |
11fdf7f2 TL |
250 | } |
251 | ||
252 | } //namespace rocksdb | |
253 | ||
254 | int main(int argc, char** argv) { | |
255 | ::testing::InitGoogleTest(&argc, argv); | |
256 | return RUN_ALL_TESTS(); | |
257 | } | |
258 | ||
259 | #else | |
260 | #include <stdio.h> | |
261 | ||
262 | int main(int /*argc*/, char** /*argv*/) { | |
263 | fprintf(stderr, | |
264 | "SKIPPED as DBImpl::DeleteFile is not supported in ROCKSDB_LITE\n"); | |
265 | return 0; | |
266 | } | |
267 | ||
268 | #endif // !ROCKSDB_LITE |