]>
Commit | Line | Data |
---|---|---|
7c673cae | 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. |
11fdf7f2 TL |
2 | // This source code is licensed under both the GPLv2 (found in the |
3 | // COPYING file in the root directory) and Apache 2.0 License | |
4 | // (found in the LICENSE.Apache file in the root directory). | |
7c673cae FG |
5 | // |
6 | // Copyright 2014 The LevelDB Authors. All rights reserved. | |
7 | // Use of this source code is governed by a BSD-style license that can be | |
8 | // found in the LICENSE file. See the AUTHORS file for names of contributors. | |
9 | ||
10 | // This test uses a custom Env to keep track of the state of a filesystem as of | |
11 | // the last "sync". It then checks for data loss errors by purposely dropping | |
12 | // file data (or entire files) not protected by a "sync". | |
13 | ||
20effc67 TL |
14 | #include "utilities/fault_injection_env.h" |
15 | ||
7c673cae FG |
16 | #include <functional> |
17 | #include <utility> | |
18 | ||
20effc67 | 19 | #include "util/random.h" |
f67539c2 | 20 | namespace ROCKSDB_NAMESPACE { |
7c673cae FG |
21 | |
22 | // Assume a filename, and not a directory name like "/foo/bar/" | |
23 | std::string GetDirName(const std::string filename) { | |
24 | size_t found = filename.find_last_of("/\\"); | |
25 | if (found == std::string::npos) { | |
26 | return ""; | |
27 | } else { | |
28 | return filename.substr(0, found); | |
29 | } | |
30 | } | |
31 | ||
32 | // A basic file truncation function suitable for this test. | |
33 | Status Truncate(Env* env, const std::string& filename, uint64_t length) { | |
494da23a | 34 | std::unique_ptr<SequentialFile> orig_file; |
7c673cae FG |
35 | const EnvOptions options; |
36 | Status s = env->NewSequentialFile(filename, &orig_file, options); | |
37 | if (!s.ok()) { | |
494da23a TL |
38 | fprintf(stderr, "Cannot open file %s for truncation: %s\n", |
39 | filename.c_str(), s.ToString().c_str()); | |
7c673cae FG |
40 | return s; |
41 | } | |
42 | ||
43 | std::unique_ptr<char[]> scratch(new char[length]); | |
f67539c2 | 44 | ROCKSDB_NAMESPACE::Slice result; |
7c673cae FG |
45 | s = orig_file->Read(length, &result, scratch.get()); |
46 | #ifdef OS_WIN | |
47 | orig_file.reset(); | |
48 | #endif | |
49 | if (s.ok()) { | |
50 | std::string tmp_name = GetDirName(filename) + "/truncate.tmp"; | |
494da23a | 51 | std::unique_ptr<WritableFile> tmp_file; |
7c673cae FG |
52 | s = env->NewWritableFile(tmp_name, &tmp_file, options); |
53 | if (s.ok()) { | |
54 | s = tmp_file->Append(result); | |
55 | if (s.ok()) { | |
56 | s = env->RenameFile(tmp_name, filename); | |
57 | } else { | |
58 | fprintf(stderr, "Cannot rename file %s to %s: %s\n", tmp_name.c_str(), | |
59 | filename.c_str(), s.ToString().c_str()); | |
60 | env->DeleteFile(tmp_name); | |
61 | } | |
62 | } | |
63 | } | |
64 | if (!s.ok()) { | |
65 | fprintf(stderr, "Cannot truncate file %s: %s\n", filename.c_str(), | |
66 | s.ToString().c_str()); | |
67 | } | |
68 | ||
69 | return s; | |
70 | } | |
71 | ||
72 | // Trim the tailing "/" in the end of `str` | |
73 | std::string TrimDirname(const std::string& str) { | |
74 | size_t found = str.find_last_not_of("/"); | |
75 | if (found == std::string::npos) { | |
76 | return str; | |
77 | } | |
78 | return str.substr(0, found + 1); | |
79 | } | |
80 | ||
81 | // Return pair <parent directory name, file name> of a full path. | |
82 | std::pair<std::string, std::string> GetDirAndName(const std::string& name) { | |
83 | std::string dirname = GetDirName(name); | |
84 | std::string fname = name.substr(dirname.size() + 1); | |
85 | return std::make_pair(dirname, fname); | |
86 | } | |
87 | ||
88 | Status FileState::DropUnsyncedData(Env* env) const { | |
89 | ssize_t sync_pos = pos_at_last_sync_ == -1 ? 0 : pos_at_last_sync_; | |
90 | return Truncate(env, filename_, sync_pos); | |
91 | } | |
92 | ||
93 | Status FileState::DropRandomUnsyncedData(Env* env, Random* rand) const { | |
94 | ssize_t sync_pos = pos_at_last_sync_ == -1 ? 0 : pos_at_last_sync_; | |
95 | assert(pos_ >= sync_pos); | |
96 | int range = static_cast<int>(pos_ - sync_pos); | |
97 | uint64_t truncated_size = | |
98 | static_cast<uint64_t>(sync_pos) + rand->Uniform(range); | |
99 | return Truncate(env, filename_, truncated_size); | |
100 | } | |
101 | ||
102 | Status TestDirectory::Fsync() { | |
f67539c2 TL |
103 | if (!env_->IsFilesystemActive()) { |
104 | return env_->GetError(); | |
105 | } | |
7c673cae FG |
106 | env_->SyncDir(dirname_); |
107 | return dir_->Fsync(); | |
108 | } | |
109 | ||
20effc67 TL |
110 | TestRandomAccessFile::TestRandomAccessFile( |
111 | std::unique_ptr<RandomAccessFile>&& target, FaultInjectionTestEnv* env) | |
112 | : target_(std::move(target)), env_(env) { | |
113 | assert(target_); | |
114 | assert(env_); | |
115 | } | |
116 | ||
117 | Status TestRandomAccessFile::Read(uint64_t offset, size_t n, Slice* result, | |
118 | char* scratch) const { | |
119 | assert(env_); | |
120 | if (!env_->IsFilesystemActive()) { | |
121 | return env_->GetError(); | |
122 | } | |
123 | ||
124 | assert(target_); | |
125 | return target_->Read(offset, n, result, scratch); | |
126 | } | |
127 | ||
128 | Status TestRandomAccessFile::Prefetch(uint64_t offset, size_t n) { | |
129 | assert(env_); | |
130 | if (!env_->IsFilesystemActive()) { | |
131 | return env_->GetError(); | |
132 | } | |
133 | ||
134 | assert(target_); | |
135 | return target_->Prefetch(offset, n); | |
136 | } | |
137 | ||
138 | Status TestRandomAccessFile::MultiRead(ReadRequest* reqs, size_t num_reqs) { | |
139 | assert(env_); | |
140 | if (!env_->IsFilesystemActive()) { | |
141 | const Status s = env_->GetError(); | |
142 | ||
143 | assert(reqs); | |
144 | for (size_t i = 0; i < num_reqs; ++i) { | |
145 | reqs[i].status = s; | |
146 | } | |
147 | ||
148 | return s; | |
149 | } | |
150 | ||
151 | assert(target_); | |
152 | return target_->MultiRead(reqs, num_reqs); | |
153 | } | |
154 | ||
7c673cae | 155 | TestWritableFile::TestWritableFile(const std::string& fname, |
494da23a | 156 | std::unique_ptr<WritableFile>&& f, |
7c673cae FG |
157 | FaultInjectionTestEnv* env) |
158 | : state_(fname), | |
159 | target_(std::move(f)), | |
160 | writable_file_opened_(true), | |
161 | env_(env) { | |
162 | assert(target_ != nullptr); | |
163 | state_.pos_ = 0; | |
164 | } | |
165 | ||
166 | TestWritableFile::~TestWritableFile() { | |
167 | if (writable_file_opened_) { | |
168 | Close(); | |
169 | } | |
170 | } | |
171 | ||
172 | Status TestWritableFile::Append(const Slice& data) { | |
173 | if (!env_->IsFilesystemActive()) { | |
11fdf7f2 | 174 | return env_->GetError(); |
7c673cae FG |
175 | } |
176 | Status s = target_->Append(data); | |
177 | if (s.ok()) { | |
178 | state_.pos_ += data.size(); | |
494da23a | 179 | env_->WritableFileAppended(state_); |
7c673cae FG |
180 | } |
181 | return s; | |
182 | } | |
183 | ||
184 | Status TestWritableFile::Close() { | |
185 | writable_file_opened_ = false; | |
186 | Status s = target_->Close(); | |
187 | if (s.ok()) { | |
188 | env_->WritableFileClosed(state_); | |
189 | } | |
190 | return s; | |
191 | } | |
192 | ||
193 | Status TestWritableFile::Flush() { | |
194 | Status s = target_->Flush(); | |
195 | if (s.ok() && env_->IsFilesystemActive()) { | |
196 | state_.pos_at_last_flush_ = state_.pos_; | |
197 | } | |
198 | return s; | |
199 | } | |
200 | ||
201 | Status TestWritableFile::Sync() { | |
202 | if (!env_->IsFilesystemActive()) { | |
203 | return Status::IOError("FaultInjectionTestEnv: not active"); | |
204 | } | |
205 | // No need to actual sync. | |
206 | state_.pos_at_last_sync_ = state_.pos_; | |
494da23a | 207 | env_->WritableFileSynced(state_); |
7c673cae FG |
208 | return Status::OK(); |
209 | } | |
210 | ||
f67539c2 TL |
211 | TestRandomRWFile::TestRandomRWFile(const std::string& /*fname*/, |
212 | std::unique_ptr<RandomRWFile>&& f, | |
213 | FaultInjectionTestEnv* env) | |
214 | : target_(std::move(f)), file_opened_(true), env_(env) { | |
215 | assert(target_ != nullptr); | |
216 | } | |
217 | ||
218 | TestRandomRWFile::~TestRandomRWFile() { | |
219 | if (file_opened_) { | |
220 | Close(); | |
221 | } | |
222 | } | |
223 | ||
224 | Status TestRandomRWFile::Write(uint64_t offset, const Slice& data) { | |
225 | if (!env_->IsFilesystemActive()) { | |
226 | return env_->GetError(); | |
227 | } | |
228 | return target_->Write(offset, data); | |
229 | } | |
230 | ||
231 | Status TestRandomRWFile::Read(uint64_t offset, size_t n, Slice* result, | |
232 | char* scratch) const { | |
233 | if (!env_->IsFilesystemActive()) { | |
234 | return env_->GetError(); | |
235 | } | |
236 | return target_->Read(offset, n, result, scratch); | |
237 | } | |
238 | ||
239 | Status TestRandomRWFile::Close() { | |
240 | file_opened_ = false; | |
241 | return target_->Close(); | |
242 | } | |
243 | ||
244 | Status TestRandomRWFile::Flush() { | |
245 | if (!env_->IsFilesystemActive()) { | |
246 | return env_->GetError(); | |
247 | } | |
248 | return target_->Flush(); | |
249 | } | |
250 | ||
251 | Status TestRandomRWFile::Sync() { | |
252 | if (!env_->IsFilesystemActive()) { | |
253 | return env_->GetError(); | |
254 | } | |
255 | return target_->Sync(); | |
256 | } | |
257 | ||
7c673cae | 258 | Status FaultInjectionTestEnv::NewDirectory(const std::string& name, |
494da23a TL |
259 | std::unique_ptr<Directory>* result) { |
260 | std::unique_ptr<Directory> r; | |
7c673cae FG |
261 | Status s = target()->NewDirectory(name, &r); |
262 | assert(s.ok()); | |
263 | if (!s.ok()) { | |
264 | return s; | |
265 | } | |
266 | result->reset(new TestDirectory(this, TrimDirname(name), r.release())); | |
267 | return Status::OK(); | |
268 | } | |
269 | ||
494da23a TL |
270 | Status FaultInjectionTestEnv::NewWritableFile( |
271 | const std::string& fname, std::unique_ptr<WritableFile>* result, | |
272 | const EnvOptions& soptions) { | |
7c673cae | 273 | if (!IsFilesystemActive()) { |
11fdf7f2 | 274 | return GetError(); |
7c673cae FG |
275 | } |
276 | // Not allow overwriting files | |
277 | Status s = target()->FileExists(fname); | |
278 | if (s.ok()) { | |
279 | return Status::Corruption("File already exists."); | |
280 | } else if (!s.IsNotFound()) { | |
281 | assert(s.IsIOError()); | |
282 | return s; | |
283 | } | |
284 | s = target()->NewWritableFile(fname, result, soptions); | |
285 | if (s.ok()) { | |
286 | result->reset(new TestWritableFile(fname, std::move(*result), this)); | |
287 | // WritableFileWriter* file is opened | |
288 | // again then it will be truncated - so forget our saved state. | |
289 | UntrackFile(fname); | |
290 | MutexLock l(&mutex_); | |
291 | open_files_.insert(fname); | |
292 | auto dir_and_name = GetDirAndName(fname); | |
293 | auto& list = dir_to_new_files_since_last_sync_[dir_and_name.first]; | |
294 | list.insert(dir_and_name.second); | |
295 | } | |
296 | return s; | |
297 | } | |
298 | ||
494da23a TL |
299 | Status FaultInjectionTestEnv::ReopenWritableFile( |
300 | const std::string& fname, std::unique_ptr<WritableFile>* result, | |
301 | const EnvOptions& soptions) { | |
302 | if (!IsFilesystemActive()) { | |
303 | return GetError(); | |
304 | } | |
305 | Status s = target()->ReopenWritableFile(fname, result, soptions); | |
306 | if (s.ok()) { | |
307 | result->reset(new TestWritableFile(fname, std::move(*result), this)); | |
308 | // WritableFileWriter* file is opened | |
309 | // again then it will be truncated - so forget our saved state. | |
310 | UntrackFile(fname); | |
311 | MutexLock l(&mutex_); | |
312 | open_files_.insert(fname); | |
313 | auto dir_and_name = GetDirAndName(fname); | |
314 | auto& list = dir_to_new_files_since_last_sync_[dir_and_name.first]; | |
315 | list.insert(dir_and_name.second); | |
316 | } | |
317 | return s; | |
318 | } | |
319 | ||
f67539c2 TL |
320 | Status FaultInjectionTestEnv::NewRandomRWFile( |
321 | const std::string& fname, std::unique_ptr<RandomRWFile>* result, | |
322 | const EnvOptions& soptions) { | |
323 | if (!IsFilesystemActive()) { | |
324 | return GetError(); | |
325 | } | |
326 | Status s = target()->NewRandomRWFile(fname, result, soptions); | |
327 | if (s.ok()) { | |
328 | result->reset(new TestRandomRWFile(fname, std::move(*result), this)); | |
329 | // WritableFileWriter* file is opened | |
330 | // again then it will be truncated - so forget our saved state. | |
331 | UntrackFile(fname); | |
332 | MutexLock l(&mutex_); | |
333 | open_files_.insert(fname); | |
334 | auto dir_and_name = GetDirAndName(fname); | |
335 | auto& list = dir_to_new_files_since_last_sync_[dir_and_name.first]; | |
336 | list.insert(dir_and_name.second); | |
337 | } | |
338 | return s; | |
339 | } | |
340 | ||
11fdf7f2 TL |
341 | Status FaultInjectionTestEnv::NewRandomAccessFile( |
342 | const std::string& fname, std::unique_ptr<RandomAccessFile>* result, | |
343 | const EnvOptions& soptions) { | |
344 | if (!IsFilesystemActive()) { | |
345 | return GetError(); | |
346 | } | |
20effc67 TL |
347 | |
348 | assert(target()); | |
349 | const Status s = target()->NewRandomAccessFile(fname, result, soptions); | |
350 | if (!s.ok()) { | |
351 | return s; | |
352 | } | |
353 | ||
354 | assert(result); | |
355 | result->reset(new TestRandomAccessFile(std::move(*result), this)); | |
356 | ||
357 | return Status::OK(); | |
11fdf7f2 TL |
358 | } |
359 | ||
7c673cae FG |
360 | Status FaultInjectionTestEnv::DeleteFile(const std::string& f) { |
361 | if (!IsFilesystemActive()) { | |
11fdf7f2 | 362 | return GetError(); |
7c673cae FG |
363 | } |
364 | Status s = EnvWrapper::DeleteFile(f); | |
7c673cae FG |
365 | if (s.ok()) { |
366 | UntrackFile(f); | |
367 | } | |
368 | return s; | |
369 | } | |
370 | ||
371 | Status FaultInjectionTestEnv::RenameFile(const std::string& s, | |
372 | const std::string& t) { | |
373 | if (!IsFilesystemActive()) { | |
11fdf7f2 | 374 | return GetError(); |
7c673cae FG |
375 | } |
376 | Status ret = EnvWrapper::RenameFile(s, t); | |
377 | ||
378 | if (ret.ok()) { | |
379 | MutexLock l(&mutex_); | |
380 | if (db_file_state_.find(s) != db_file_state_.end()) { | |
381 | db_file_state_[t] = db_file_state_[s]; | |
382 | db_file_state_.erase(s); | |
383 | } | |
384 | ||
385 | auto sdn = GetDirAndName(s); | |
386 | auto tdn = GetDirAndName(t); | |
387 | if (dir_to_new_files_since_last_sync_[sdn.first].erase(sdn.second) != 0) { | |
388 | auto& tlist = dir_to_new_files_since_last_sync_[tdn.first]; | |
389 | assert(tlist.find(tdn.second) == tlist.end()); | |
390 | tlist.insert(tdn.second); | |
391 | } | |
392 | } | |
393 | ||
394 | return ret; | |
395 | } | |
396 | ||
397 | void FaultInjectionTestEnv::WritableFileClosed(const FileState& state) { | |
398 | MutexLock l(&mutex_); | |
399 | if (open_files_.find(state.filename_) != open_files_.end()) { | |
400 | db_file_state_[state.filename_] = state; | |
401 | open_files_.erase(state.filename_); | |
402 | } | |
403 | } | |
404 | ||
494da23a TL |
405 | void FaultInjectionTestEnv::WritableFileSynced(const FileState& state) { |
406 | MutexLock l(&mutex_); | |
407 | if (open_files_.find(state.filename_) != open_files_.end()) { | |
408 | if (db_file_state_.find(state.filename_) == db_file_state_.end()) { | |
409 | db_file_state_.insert(std::make_pair(state.filename_, state)); | |
410 | } else { | |
411 | db_file_state_[state.filename_] = state; | |
412 | } | |
413 | } | |
414 | } | |
415 | ||
416 | void FaultInjectionTestEnv::WritableFileAppended(const FileState& state) { | |
417 | MutexLock l(&mutex_); | |
418 | if (open_files_.find(state.filename_) != open_files_.end()) { | |
419 | if (db_file_state_.find(state.filename_) == db_file_state_.end()) { | |
420 | db_file_state_.insert(std::make_pair(state.filename_, state)); | |
421 | } else { | |
422 | db_file_state_[state.filename_] = state; | |
423 | } | |
424 | } | |
425 | } | |
426 | ||
7c673cae FG |
427 | // For every file that is not fully synced, make a call to `func` with |
428 | // FileState of the file as the parameter. | |
429 | Status FaultInjectionTestEnv::DropFileData( | |
430 | std::function<Status(Env*, FileState)> func) { | |
431 | Status s; | |
432 | MutexLock l(&mutex_); | |
433 | for (std::map<std::string, FileState>::const_iterator it = | |
434 | db_file_state_.begin(); | |
435 | s.ok() && it != db_file_state_.end(); ++it) { | |
436 | const FileState& state = it->second; | |
437 | if (!state.IsFullySynced()) { | |
438 | s = func(target(), state); | |
439 | } | |
440 | } | |
441 | return s; | |
442 | } | |
443 | ||
444 | Status FaultInjectionTestEnv::DropUnsyncedFileData() { | |
445 | return DropFileData([&](Env* env, const FileState& state) { | |
446 | return state.DropUnsyncedData(env); | |
447 | }); | |
448 | } | |
449 | ||
450 | Status FaultInjectionTestEnv::DropRandomUnsyncedFileData(Random* rnd) { | |
451 | return DropFileData([&](Env* env, const FileState& state) { | |
452 | return state.DropRandomUnsyncedData(env, rnd); | |
453 | }); | |
454 | } | |
455 | ||
456 | Status FaultInjectionTestEnv::DeleteFilesCreatedAfterLastDirSync() { | |
457 | // Because DeleteFile access this container make a copy to avoid deadlock | |
458 | std::map<std::string, std::set<std::string>> map_copy; | |
459 | { | |
460 | MutexLock l(&mutex_); | |
461 | map_copy.insert(dir_to_new_files_since_last_sync_.begin(), | |
462 | dir_to_new_files_since_last_sync_.end()); | |
463 | } | |
464 | ||
465 | for (auto& pair : map_copy) { | |
466 | for (std::string name : pair.second) { | |
467 | Status s = DeleteFile(pair.first + "/" + name); | |
468 | if (!s.ok()) { | |
469 | return s; | |
470 | } | |
471 | } | |
472 | } | |
473 | return Status::OK(); | |
474 | } | |
475 | void FaultInjectionTestEnv::ResetState() { | |
476 | MutexLock l(&mutex_); | |
477 | db_file_state_.clear(); | |
478 | dir_to_new_files_since_last_sync_.clear(); | |
479 | SetFilesystemActiveNoLock(true); | |
480 | } | |
481 | ||
482 | void FaultInjectionTestEnv::UntrackFile(const std::string& f) { | |
483 | MutexLock l(&mutex_); | |
484 | auto dir_and_name = GetDirAndName(f); | |
485 | dir_to_new_files_since_last_sync_[dir_and_name.first].erase( | |
486 | dir_and_name.second); | |
487 | db_file_state_.erase(f); | |
488 | open_files_.erase(f); | |
489 | } | |
f67539c2 | 490 | } // namespace ROCKSDB_NAMESPACE |