]>
Commit | Line | Data |
---|---|---|
7c673cae | 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. |
11fdf7f2 TL |
2 | // This source code is licensed under both the GPLv2 (found in the |
3 | // COPYING file in the root directory) and Apache 2.0 License | |
4 | // (found in the LICENSE.Apache file in the root directory). | |
7c673cae FG |
5 | // |
6 | // Copyright 2014 The LevelDB Authors. All rights reserved. | |
7 | // Use of this source code is governed by a BSD-style license that can be | |
8 | // found in the LICENSE file. See the AUTHORS file for names of contributors. | |
9 | ||
10 | // This test uses a custom Env to keep track of the state of a filesystem as of | |
11 | // the last "sync". It then checks for data loss errors by purposely dropping | |
12 | // file data (or entire files) not protected by a "sync". | |
13 | ||
14 | #include "util/fault_injection_test_env.h" | |
15 | #include <functional> | |
16 | #include <utility> | |
17 | ||
18 | namespace rocksdb { | |
19 | ||
20 | // Assume a filename, and not a directory name like "/foo/bar/" | |
21 | std::string GetDirName(const std::string filename) { | |
22 | size_t found = filename.find_last_of("/\\"); | |
23 | if (found == std::string::npos) { | |
24 | return ""; | |
25 | } else { | |
26 | return filename.substr(0, found); | |
27 | } | |
28 | } | |
29 | ||
30 | // A basic file truncation function suitable for this test. | |
31 | Status Truncate(Env* env, const std::string& filename, uint64_t length) { | |
494da23a | 32 | std::unique_ptr<SequentialFile> orig_file; |
7c673cae FG |
33 | const EnvOptions options; |
34 | Status s = env->NewSequentialFile(filename, &orig_file, options); | |
35 | if (!s.ok()) { | |
494da23a TL |
36 | fprintf(stderr, "Cannot open file %s for truncation: %s\n", |
37 | filename.c_str(), s.ToString().c_str()); | |
7c673cae FG |
38 | return s; |
39 | } | |
40 | ||
41 | std::unique_ptr<char[]> scratch(new char[length]); | |
42 | rocksdb::Slice result; | |
43 | s = orig_file->Read(length, &result, scratch.get()); | |
44 | #ifdef OS_WIN | |
45 | orig_file.reset(); | |
46 | #endif | |
47 | if (s.ok()) { | |
48 | std::string tmp_name = GetDirName(filename) + "/truncate.tmp"; | |
494da23a | 49 | std::unique_ptr<WritableFile> tmp_file; |
7c673cae FG |
50 | s = env->NewWritableFile(tmp_name, &tmp_file, options); |
51 | if (s.ok()) { | |
52 | s = tmp_file->Append(result); | |
53 | if (s.ok()) { | |
54 | s = env->RenameFile(tmp_name, filename); | |
55 | } else { | |
56 | fprintf(stderr, "Cannot rename file %s to %s: %s\n", tmp_name.c_str(), | |
57 | filename.c_str(), s.ToString().c_str()); | |
58 | env->DeleteFile(tmp_name); | |
59 | } | |
60 | } | |
61 | } | |
62 | if (!s.ok()) { | |
63 | fprintf(stderr, "Cannot truncate file %s: %s\n", filename.c_str(), | |
64 | s.ToString().c_str()); | |
65 | } | |
66 | ||
67 | return s; | |
68 | } | |
69 | ||
70 | // Trim the tailing "/" in the end of `str` | |
71 | std::string TrimDirname(const std::string& str) { | |
72 | size_t found = str.find_last_not_of("/"); | |
73 | if (found == std::string::npos) { | |
74 | return str; | |
75 | } | |
76 | return str.substr(0, found + 1); | |
77 | } | |
78 | ||
79 | // Return pair <parent directory name, file name> of a full path. | |
80 | std::pair<std::string, std::string> GetDirAndName(const std::string& name) { | |
81 | std::string dirname = GetDirName(name); | |
82 | std::string fname = name.substr(dirname.size() + 1); | |
83 | return std::make_pair(dirname, fname); | |
84 | } | |
85 | ||
86 | Status FileState::DropUnsyncedData(Env* env) const { | |
87 | ssize_t sync_pos = pos_at_last_sync_ == -1 ? 0 : pos_at_last_sync_; | |
88 | return Truncate(env, filename_, sync_pos); | |
89 | } | |
90 | ||
91 | Status FileState::DropRandomUnsyncedData(Env* env, Random* rand) const { | |
92 | ssize_t sync_pos = pos_at_last_sync_ == -1 ? 0 : pos_at_last_sync_; | |
93 | assert(pos_ >= sync_pos); | |
94 | int range = static_cast<int>(pos_ - sync_pos); | |
95 | uint64_t truncated_size = | |
96 | static_cast<uint64_t>(sync_pos) + rand->Uniform(range); | |
97 | return Truncate(env, filename_, truncated_size); | |
98 | } | |
99 | ||
100 | Status TestDirectory::Fsync() { | |
101 | env_->SyncDir(dirname_); | |
102 | return dir_->Fsync(); | |
103 | } | |
104 | ||
105 | TestWritableFile::TestWritableFile(const std::string& fname, | |
494da23a | 106 | std::unique_ptr<WritableFile>&& f, |
7c673cae FG |
107 | FaultInjectionTestEnv* env) |
108 | : state_(fname), | |
109 | target_(std::move(f)), | |
110 | writable_file_opened_(true), | |
111 | env_(env) { | |
112 | assert(target_ != nullptr); | |
113 | state_.pos_ = 0; | |
114 | } | |
115 | ||
116 | TestWritableFile::~TestWritableFile() { | |
117 | if (writable_file_opened_) { | |
118 | Close(); | |
119 | } | |
120 | } | |
121 | ||
122 | Status TestWritableFile::Append(const Slice& data) { | |
123 | if (!env_->IsFilesystemActive()) { | |
11fdf7f2 | 124 | return env_->GetError(); |
7c673cae FG |
125 | } |
126 | Status s = target_->Append(data); | |
127 | if (s.ok()) { | |
128 | state_.pos_ += data.size(); | |
494da23a | 129 | env_->WritableFileAppended(state_); |
7c673cae FG |
130 | } |
131 | return s; | |
132 | } | |
133 | ||
134 | Status TestWritableFile::Close() { | |
135 | writable_file_opened_ = false; | |
136 | Status s = target_->Close(); | |
137 | if (s.ok()) { | |
138 | env_->WritableFileClosed(state_); | |
139 | } | |
140 | return s; | |
141 | } | |
142 | ||
143 | Status TestWritableFile::Flush() { | |
144 | Status s = target_->Flush(); | |
145 | if (s.ok() && env_->IsFilesystemActive()) { | |
146 | state_.pos_at_last_flush_ = state_.pos_; | |
147 | } | |
148 | return s; | |
149 | } | |
150 | ||
151 | Status TestWritableFile::Sync() { | |
152 | if (!env_->IsFilesystemActive()) { | |
153 | return Status::IOError("FaultInjectionTestEnv: not active"); | |
154 | } | |
155 | // No need to actual sync. | |
156 | state_.pos_at_last_sync_ = state_.pos_; | |
494da23a | 157 | env_->WritableFileSynced(state_); |
7c673cae FG |
158 | return Status::OK(); |
159 | } | |
160 | ||
161 | Status FaultInjectionTestEnv::NewDirectory(const std::string& name, | |
494da23a TL |
162 | std::unique_ptr<Directory>* result) { |
163 | std::unique_ptr<Directory> r; | |
7c673cae FG |
164 | Status s = target()->NewDirectory(name, &r); |
165 | assert(s.ok()); | |
166 | if (!s.ok()) { | |
167 | return s; | |
168 | } | |
169 | result->reset(new TestDirectory(this, TrimDirname(name), r.release())); | |
170 | return Status::OK(); | |
171 | } | |
172 | ||
494da23a TL |
173 | Status FaultInjectionTestEnv::NewWritableFile( |
174 | const std::string& fname, std::unique_ptr<WritableFile>* result, | |
175 | const EnvOptions& soptions) { | |
7c673cae | 176 | if (!IsFilesystemActive()) { |
11fdf7f2 | 177 | return GetError(); |
7c673cae FG |
178 | } |
179 | // Not allow overwriting files | |
180 | Status s = target()->FileExists(fname); | |
181 | if (s.ok()) { | |
182 | return Status::Corruption("File already exists."); | |
183 | } else if (!s.IsNotFound()) { | |
184 | assert(s.IsIOError()); | |
185 | return s; | |
186 | } | |
187 | s = target()->NewWritableFile(fname, result, soptions); | |
188 | if (s.ok()) { | |
189 | result->reset(new TestWritableFile(fname, std::move(*result), this)); | |
190 | // WritableFileWriter* file is opened | |
191 | // again then it will be truncated - so forget our saved state. | |
192 | UntrackFile(fname); | |
193 | MutexLock l(&mutex_); | |
194 | open_files_.insert(fname); | |
195 | auto dir_and_name = GetDirAndName(fname); | |
196 | auto& list = dir_to_new_files_since_last_sync_[dir_and_name.first]; | |
197 | list.insert(dir_and_name.second); | |
198 | } | |
199 | return s; | |
200 | } | |
201 | ||
494da23a TL |
202 | Status FaultInjectionTestEnv::ReopenWritableFile( |
203 | const std::string& fname, std::unique_ptr<WritableFile>* result, | |
204 | const EnvOptions& soptions) { | |
205 | if (!IsFilesystemActive()) { | |
206 | return GetError(); | |
207 | } | |
208 | Status s = target()->ReopenWritableFile(fname, result, soptions); | |
209 | if (s.ok()) { | |
210 | result->reset(new TestWritableFile(fname, std::move(*result), this)); | |
211 | // WritableFileWriter* file is opened | |
212 | // again then it will be truncated - so forget our saved state. | |
213 | UntrackFile(fname); | |
214 | MutexLock l(&mutex_); | |
215 | open_files_.insert(fname); | |
216 | auto dir_and_name = GetDirAndName(fname); | |
217 | auto& list = dir_to_new_files_since_last_sync_[dir_and_name.first]; | |
218 | list.insert(dir_and_name.second); | |
219 | } | |
220 | return s; | |
221 | } | |
222 | ||
11fdf7f2 TL |
223 | Status FaultInjectionTestEnv::NewRandomAccessFile( |
224 | const std::string& fname, std::unique_ptr<RandomAccessFile>* result, | |
225 | const EnvOptions& soptions) { | |
226 | if (!IsFilesystemActive()) { | |
227 | return GetError(); | |
228 | } | |
229 | return target()->NewRandomAccessFile(fname, result, soptions); | |
230 | } | |
231 | ||
7c673cae FG |
232 | Status FaultInjectionTestEnv::DeleteFile(const std::string& f) { |
233 | if (!IsFilesystemActive()) { | |
11fdf7f2 | 234 | return GetError(); |
7c673cae FG |
235 | } |
236 | Status s = EnvWrapper::DeleteFile(f); | |
237 | if (!s.ok()) { | |
238 | fprintf(stderr, "Cannot delete file %s: %s\n", f.c_str(), | |
239 | s.ToString().c_str()); | |
240 | } | |
241 | assert(s.ok()); | |
242 | if (s.ok()) { | |
243 | UntrackFile(f); | |
244 | } | |
245 | return s; | |
246 | } | |
247 | ||
248 | Status FaultInjectionTestEnv::RenameFile(const std::string& s, | |
249 | const std::string& t) { | |
250 | if (!IsFilesystemActive()) { | |
11fdf7f2 | 251 | return GetError(); |
7c673cae FG |
252 | } |
253 | Status ret = EnvWrapper::RenameFile(s, t); | |
254 | ||
255 | if (ret.ok()) { | |
256 | MutexLock l(&mutex_); | |
257 | if (db_file_state_.find(s) != db_file_state_.end()) { | |
258 | db_file_state_[t] = db_file_state_[s]; | |
259 | db_file_state_.erase(s); | |
260 | } | |
261 | ||
262 | auto sdn = GetDirAndName(s); | |
263 | auto tdn = GetDirAndName(t); | |
264 | if (dir_to_new_files_since_last_sync_[sdn.first].erase(sdn.second) != 0) { | |
265 | auto& tlist = dir_to_new_files_since_last_sync_[tdn.first]; | |
266 | assert(tlist.find(tdn.second) == tlist.end()); | |
267 | tlist.insert(tdn.second); | |
268 | } | |
269 | } | |
270 | ||
271 | return ret; | |
272 | } | |
273 | ||
274 | void FaultInjectionTestEnv::WritableFileClosed(const FileState& state) { | |
275 | MutexLock l(&mutex_); | |
276 | if (open_files_.find(state.filename_) != open_files_.end()) { | |
277 | db_file_state_[state.filename_] = state; | |
278 | open_files_.erase(state.filename_); | |
279 | } | |
280 | } | |
281 | ||
494da23a TL |
282 | void FaultInjectionTestEnv::WritableFileSynced(const FileState& state) { |
283 | MutexLock l(&mutex_); | |
284 | if (open_files_.find(state.filename_) != open_files_.end()) { | |
285 | if (db_file_state_.find(state.filename_) == db_file_state_.end()) { | |
286 | db_file_state_.insert(std::make_pair(state.filename_, state)); | |
287 | } else { | |
288 | db_file_state_[state.filename_] = state; | |
289 | } | |
290 | } | |
291 | } | |
292 | ||
293 | void FaultInjectionTestEnv::WritableFileAppended(const FileState& state) { | |
294 | MutexLock l(&mutex_); | |
295 | if (open_files_.find(state.filename_) != open_files_.end()) { | |
296 | if (db_file_state_.find(state.filename_) == db_file_state_.end()) { | |
297 | db_file_state_.insert(std::make_pair(state.filename_, state)); | |
298 | } else { | |
299 | db_file_state_[state.filename_] = state; | |
300 | } | |
301 | } | |
302 | } | |
303 | ||
7c673cae FG |
304 | // For every file that is not fully synced, make a call to `func` with |
305 | // FileState of the file as the parameter. | |
306 | Status FaultInjectionTestEnv::DropFileData( | |
307 | std::function<Status(Env*, FileState)> func) { | |
308 | Status s; | |
309 | MutexLock l(&mutex_); | |
310 | for (std::map<std::string, FileState>::const_iterator it = | |
311 | db_file_state_.begin(); | |
312 | s.ok() && it != db_file_state_.end(); ++it) { | |
313 | const FileState& state = it->second; | |
314 | if (!state.IsFullySynced()) { | |
315 | s = func(target(), state); | |
316 | } | |
317 | } | |
318 | return s; | |
319 | } | |
320 | ||
321 | Status FaultInjectionTestEnv::DropUnsyncedFileData() { | |
322 | return DropFileData([&](Env* env, const FileState& state) { | |
323 | return state.DropUnsyncedData(env); | |
324 | }); | |
325 | } | |
326 | ||
327 | Status FaultInjectionTestEnv::DropRandomUnsyncedFileData(Random* rnd) { | |
328 | return DropFileData([&](Env* env, const FileState& state) { | |
329 | return state.DropRandomUnsyncedData(env, rnd); | |
330 | }); | |
331 | } | |
332 | ||
333 | Status FaultInjectionTestEnv::DeleteFilesCreatedAfterLastDirSync() { | |
334 | // Because DeleteFile access this container make a copy to avoid deadlock | |
335 | std::map<std::string, std::set<std::string>> map_copy; | |
336 | { | |
337 | MutexLock l(&mutex_); | |
338 | map_copy.insert(dir_to_new_files_since_last_sync_.begin(), | |
339 | dir_to_new_files_since_last_sync_.end()); | |
340 | } | |
341 | ||
342 | for (auto& pair : map_copy) { | |
343 | for (std::string name : pair.second) { | |
344 | Status s = DeleteFile(pair.first + "/" + name); | |
345 | if (!s.ok()) { | |
346 | return s; | |
347 | } | |
348 | } | |
349 | } | |
350 | return Status::OK(); | |
351 | } | |
352 | void FaultInjectionTestEnv::ResetState() { | |
353 | MutexLock l(&mutex_); | |
354 | db_file_state_.clear(); | |
355 | dir_to_new_files_since_last_sync_.clear(); | |
356 | SetFilesystemActiveNoLock(true); | |
357 | } | |
358 | ||
359 | void FaultInjectionTestEnv::UntrackFile(const std::string& f) { | |
360 | MutexLock l(&mutex_); | |
361 | auto dir_and_name = GetDirAndName(f); | |
362 | dir_to_new_files_since_last_sync_[dir_and_name.first].erase( | |
363 | dir_and_name.second); | |
364 | db_file_state_.erase(f); | |
365 | open_files_.erase(f); | |
366 | } | |
367 | } // namespace rocksdb |