]>
Commit | Line | Data |
---|---|---|
20effc67 TL |
1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. |
2 | // This source code is licensed under both the GPLv2 (found in the | |
3 | // COPYING file in the root directory) and Apache 2.0 License | |
4 | // (found in the LICENSE.Apache file in the root directory). | |
5 | // | |
6 | // Copyright 2014 The LevelDB Authors. All rights reserved. | |
7 | // Use of this source code is governed by a BSD-style license that can be | |
8 | // found in the LICENSE file. See the AUTHORS file for names of contributors. | |
9 | ||
10 | // This test uses a custom FileSystem to keep track of the state of a file | |
11 | // system the last "Sync". The data being written is cached in a "buffer". | |
12 | // Only when "Sync" is called, the data will be persistent. It can similate | |
13 | // file data loss (or entire files) not protected by a "Sync". For any of the | |
14 | // FileSystem related operations, by specify the "IOStatus Error", a specific | |
15 | // error can be returned when file system is not activated. | |
16 | ||
17 | #pragma once | |
18 | ||
19 | #include <algorithm> | |
20 | #include <map> | |
21 | #include <set> | |
22 | #include <string> | |
23 | ||
24 | #include "file/filename.h" | |
1e59de90 | 25 | #include "rocksdb/file_system.h" |
20effc67 TL |
26 | #include "util/mutexlock.h" |
27 | #include "util/random.h" | |
28 | #include "util/thread_local.h" | |
29 | ||
30 | namespace ROCKSDB_NAMESPACE { | |
31 | ||
32 | class TestFSWritableFile; | |
33 | class FaultInjectionTestFS; | |
34 | ||
35 | struct FSFileState { | |
36 | std::string filename_; | |
37 | ssize_t pos_; | |
38 | ssize_t pos_at_last_sync_; | |
39 | ssize_t pos_at_last_flush_; | |
40 | std::string buffer_; | |
41 | ||
42 | explicit FSFileState(const std::string& filename) | |
43 | : filename_(filename), | |
44 | pos_(-1), | |
45 | pos_at_last_sync_(-1), | |
46 | pos_at_last_flush_(-1) {} | |
47 | ||
48 | FSFileState() : pos_(-1), pos_at_last_sync_(-1), pos_at_last_flush_(-1) {} | |
49 | ||
50 | bool IsFullySynced() const { return pos_ <= 0 || pos_ == pos_at_last_sync_; } | |
51 | ||
52 | IOStatus DropUnsyncedData(); | |
53 | ||
54 | IOStatus DropRandomUnsyncedData(Random* rand); | |
55 | }; | |
56 | ||
57 | // A wrapper around WritableFileWriter* file | |
58 | // is written to or sync'ed. | |
59 | class TestFSWritableFile : public FSWritableFile { | |
60 | public: | |
61 | explicit TestFSWritableFile(const std::string& fname, | |
1e59de90 | 62 | const FileOptions& file_opts, |
20effc67 TL |
63 | std::unique_ptr<FSWritableFile>&& f, |
64 | FaultInjectionTestFS* fs); | |
65 | virtual ~TestFSWritableFile(); | |
66 | virtual IOStatus Append(const Slice& data, const IOOptions&, | |
67 | IODebugContext*) override; | |
68 | virtual IOStatus Append(const Slice& data, const IOOptions& options, | |
1e59de90 TL |
69 | const DataVerificationInfo& verification_info, |
70 | IODebugContext* dbg) override; | |
20effc67 TL |
71 | virtual IOStatus Truncate(uint64_t size, const IOOptions& options, |
72 | IODebugContext* dbg) override { | |
73 | return target_->Truncate(size, options, dbg); | |
74 | } | |
75 | virtual IOStatus Close(const IOOptions& options, | |
76 | IODebugContext* dbg) override; | |
77 | virtual IOStatus Flush(const IOOptions&, IODebugContext*) override; | |
78 | virtual IOStatus Sync(const IOOptions& options, IODebugContext* dbg) override; | |
1e59de90 TL |
79 | virtual IOStatus RangeSync(uint64_t /*offset*/, uint64_t /*nbytes*/, |
80 | const IOOptions& options, | |
81 | IODebugContext* dbg) override; | |
20effc67 TL |
82 | virtual bool IsSyncThreadSafe() const override { return true; } |
83 | virtual IOStatus PositionedAppend(const Slice& data, uint64_t offset, | |
84 | const IOOptions& options, | |
85 | IODebugContext* dbg) override { | |
86 | return target_->PositionedAppend(data, offset, options, dbg); | |
87 | } | |
88 | IOStatus PositionedAppend(const Slice& data, uint64_t offset, | |
89 | const IOOptions& options, | |
1e59de90 TL |
90 | const DataVerificationInfo& verification_info, |
91 | IODebugContext* dbg) override; | |
20effc67 TL |
92 | virtual size_t GetRequiredBufferAlignment() const override { |
93 | return target_->GetRequiredBufferAlignment(); | |
94 | } | |
95 | virtual bool use_direct_io() const override { | |
96 | return target_->use_direct_io(); | |
97 | }; | |
98 | ||
99 | private: | |
1e59de90 TL |
100 | FSFileState state_; // Need protection by mutex_ |
101 | FileOptions file_opts_; | |
20effc67 TL |
102 | std::unique_ptr<FSWritableFile> target_; |
103 | bool writable_file_opened_; | |
104 | FaultInjectionTestFS* fs_; | |
105 | port::Mutex mutex_; | |
106 | }; | |
107 | ||
108 | // A wrapper around WritableFileWriter* file | |
109 | // is written to or sync'ed. | |
110 | class TestFSRandomRWFile : public FSRandomRWFile { | |
111 | public: | |
112 | explicit TestFSRandomRWFile(const std::string& fname, | |
113 | std::unique_ptr<FSRandomRWFile>&& f, | |
114 | FaultInjectionTestFS* fs); | |
115 | virtual ~TestFSRandomRWFile(); | |
116 | IOStatus Write(uint64_t offset, const Slice& data, const IOOptions& options, | |
117 | IODebugContext* dbg) override; | |
118 | IOStatus Read(uint64_t offset, size_t n, const IOOptions& options, | |
119 | Slice* result, char* scratch, | |
120 | IODebugContext* dbg) const override; | |
121 | IOStatus Close(const IOOptions& options, IODebugContext* dbg) override; | |
122 | IOStatus Flush(const IOOptions& options, IODebugContext* dbg) override; | |
123 | IOStatus Sync(const IOOptions& options, IODebugContext* dbg) override; | |
124 | size_t GetRequiredBufferAlignment() const override { | |
125 | return target_->GetRequiredBufferAlignment(); | |
126 | } | |
127 | bool use_direct_io() const override { return target_->use_direct_io(); }; | |
128 | ||
129 | private: | |
130 | std::unique_ptr<FSRandomRWFile> target_; | |
131 | bool file_opened_; | |
132 | FaultInjectionTestFS* fs_; | |
133 | }; | |
134 | ||
135 | class TestFSRandomAccessFile : public FSRandomAccessFile { | |
136 | public: | |
137 | explicit TestFSRandomAccessFile(const std::string& fname, | |
1e59de90 TL |
138 | std::unique_ptr<FSRandomAccessFile>&& f, |
139 | FaultInjectionTestFS* fs); | |
20effc67 TL |
140 | ~TestFSRandomAccessFile() override {} |
141 | IOStatus Read(uint64_t offset, size_t n, const IOOptions& options, | |
142 | Slice* result, char* scratch, | |
143 | IODebugContext* dbg) const override; | |
1e59de90 TL |
144 | IOStatus MultiRead(FSReadRequest* reqs, size_t num_reqs, |
145 | const IOOptions& options, IODebugContext* dbg) override; | |
20effc67 TL |
146 | size_t GetRequiredBufferAlignment() const override { |
147 | return target_->GetRequiredBufferAlignment(); | |
148 | } | |
149 | bool use_direct_io() const override { return target_->use_direct_io(); } | |
150 | ||
1e59de90 TL |
151 | size_t GetUniqueId(char* id, size_t max_size) const override; |
152 | ||
20effc67 TL |
153 | private: |
154 | std::unique_ptr<FSRandomAccessFile> target_; | |
155 | FaultInjectionTestFS* fs_; | |
156 | }; | |
157 | ||
1e59de90 TL |
158 | class TestFSSequentialFile : public FSSequentialFileOwnerWrapper { |
159 | public: | |
160 | explicit TestFSSequentialFile(std::unique_ptr<FSSequentialFile>&& f, | |
161 | FaultInjectionTestFS* fs) | |
162 | : FSSequentialFileOwnerWrapper(std::move(f)), fs_(fs) {} | |
163 | IOStatus Read(size_t n, const IOOptions& options, Slice* result, | |
164 | char* scratch, IODebugContext* dbg) override; | |
165 | IOStatus PositionedRead(uint64_t offset, size_t n, const IOOptions& options, | |
166 | Slice* result, char* scratch, | |
167 | IODebugContext* dbg) override; | |
168 | ||
169 | private: | |
170 | FaultInjectionTestFS* fs_; | |
171 | }; | |
172 | ||
20effc67 TL |
173 | class TestFSDirectory : public FSDirectory { |
174 | public: | |
175 | explicit TestFSDirectory(FaultInjectionTestFS* fs, std::string dirname, | |
176 | FSDirectory* dir) | |
177 | : fs_(fs), dirname_(dirname), dir_(dir) {} | |
178 | ~TestFSDirectory() {} | |
179 | ||
180 | virtual IOStatus Fsync(const IOOptions& options, | |
181 | IODebugContext* dbg) override; | |
182 | ||
1e59de90 TL |
183 | virtual IOStatus Close(const IOOptions& options, |
184 | IODebugContext* dbg) override; | |
185 | ||
186 | virtual IOStatus FsyncWithDirOptions( | |
187 | const IOOptions& options, IODebugContext* dbg, | |
188 | const DirFsyncOptions& dir_fsync_options) override; | |
189 | ||
20effc67 TL |
190 | private: |
191 | FaultInjectionTestFS* fs_; | |
192 | std::string dirname_; | |
193 | std::unique_ptr<FSDirectory> dir_; | |
194 | }; | |
195 | ||
196 | class FaultInjectionTestFS : public FileSystemWrapper { | |
197 | public: | |
198 | explicit FaultInjectionTestFS(const std::shared_ptr<FileSystem>& base) | |
199 | : FileSystemWrapper(base), | |
200 | filesystem_active_(true), | |
201 | filesystem_writable_(false), | |
1e59de90 TL |
202 | thread_local_error_(new ThreadLocalPtr(DeleteThreadLocalErrorContext)), |
203 | enable_write_error_injection_(false), | |
204 | enable_metadata_write_error_injection_(false), | |
205 | write_error_rand_(0), | |
206 | write_error_one_in_(0), | |
207 | metadata_write_error_one_in_(0), | |
208 | read_error_one_in_(0), | |
209 | ingest_data_corruption_before_write_(false), | |
210 | fail_get_file_unique_id_(false) {} | |
20effc67 TL |
211 | virtual ~FaultInjectionTestFS() { error_.PermitUncheckedError(); } |
212 | ||
1e59de90 TL |
213 | static const char* kClassName() { return "FaultInjectionTestFS"; } |
214 | const char* Name() const override { return kClassName(); } | |
20effc67 TL |
215 | |
216 | IOStatus NewDirectory(const std::string& name, const IOOptions& options, | |
217 | std::unique_ptr<FSDirectory>* result, | |
218 | IODebugContext* dbg) override; | |
219 | ||
220 | IOStatus NewWritableFile(const std::string& fname, | |
221 | const FileOptions& file_opts, | |
222 | std::unique_ptr<FSWritableFile>* result, | |
223 | IODebugContext* dbg) override; | |
224 | ||
225 | IOStatus ReopenWritableFile(const std::string& fname, | |
226 | const FileOptions& file_opts, | |
227 | std::unique_ptr<FSWritableFile>* result, | |
228 | IODebugContext* dbg) override; | |
229 | ||
230 | IOStatus NewRandomRWFile(const std::string& fname, | |
231 | const FileOptions& file_opts, | |
232 | std::unique_ptr<FSRandomRWFile>* result, | |
233 | IODebugContext* dbg) override; | |
234 | ||
235 | IOStatus NewRandomAccessFile(const std::string& fname, | |
236 | const FileOptions& file_opts, | |
237 | std::unique_ptr<FSRandomAccessFile>* result, | |
238 | IODebugContext* dbg) override; | |
1e59de90 TL |
239 | IOStatus NewSequentialFile(const std::string& f, const FileOptions& file_opts, |
240 | std::unique_ptr<FSSequentialFile>* r, | |
241 | IODebugContext* dbg) override; | |
20effc67 TL |
242 | |
243 | virtual IOStatus DeleteFile(const std::string& f, const IOOptions& options, | |
244 | IODebugContext* dbg) override; | |
245 | ||
246 | virtual IOStatus RenameFile(const std::string& s, const std::string& t, | |
247 | const IOOptions& options, | |
248 | IODebugContext* dbg) override; | |
249 | ||
1e59de90 TL |
250 | virtual IOStatus LinkFile(const std::string& src, const std::string& target, |
251 | const IOOptions& options, | |
252 | IODebugContext* dbg) override; | |
253 | ||
20effc67 TL |
254 | // Undef to eliminate clash on Windows |
255 | #undef GetFreeSpace | |
256 | virtual IOStatus GetFreeSpace(const std::string& path, | |
257 | const IOOptions& options, uint64_t* disk_free, | |
258 | IODebugContext* dbg) override { | |
259 | IOStatus io_s; | |
1e59de90 TL |
260 | if (!IsFilesystemActive() && |
261 | error_.subcode() == IOStatus::SubCode::kNoSpace) { | |
20effc67 TL |
262 | *disk_free = 0; |
263 | } else { | |
264 | io_s = target()->GetFreeSpace(path, options, disk_free, dbg); | |
265 | } | |
266 | return io_s; | |
267 | } | |
268 | ||
269 | void WritableFileClosed(const FSFileState& state); | |
270 | ||
271 | void WritableFileSynced(const FSFileState& state); | |
272 | ||
273 | void WritableFileAppended(const FSFileState& state); | |
274 | ||
275 | IOStatus DropUnsyncedFileData(); | |
276 | ||
277 | IOStatus DropRandomUnsyncedFileData(Random* rnd); | |
278 | ||
279 | IOStatus DeleteFilesCreatedAfterLastDirSync(const IOOptions& options, | |
280 | IODebugContext* dbg); | |
281 | ||
282 | void ResetState(); | |
283 | ||
284 | void UntrackFile(const std::string& f); | |
285 | ||
286 | void SyncDir(const std::string& dirname) { | |
287 | MutexLock l(&mutex_); | |
288 | dir_to_new_files_since_last_sync_.erase(dirname); | |
289 | } | |
290 | ||
291 | // Setting the filesystem to inactive is the test equivalent to simulating a | |
292 | // system reset. Setting to inactive will freeze our saved filesystem state so | |
293 | // that it will stop being recorded. It can then be reset back to the state at | |
294 | // the time of the reset. | |
295 | bool IsFilesystemActive() { | |
296 | MutexLock l(&mutex_); | |
297 | return filesystem_active_; | |
298 | } | |
299 | ||
300 | // Setting filesystem_writable_ makes NewWritableFile. ReopenWritableFile, | |
301 | // and NewRandomRWFile bypass FaultInjectionTestFS and go directly to the | |
302 | // target FS | |
303 | bool IsFilesystemDirectWritable() { | |
304 | MutexLock l(&mutex_); | |
305 | return filesystem_writable_; | |
306 | } | |
1e59de90 TL |
307 | bool ShouldUseDiretWritable(const std::string& file_name) { |
308 | MutexLock l(&mutex_); | |
309 | if (filesystem_writable_) { | |
310 | return true; | |
311 | } | |
312 | FileType file_type = kTempFile; | |
313 | uint64_t file_number = 0; | |
314 | if (!TryParseFileName(file_name, &file_number, &file_type)) { | |
315 | return false; | |
316 | } | |
317 | return skip_direct_writable_types_.find(file_type) != | |
318 | skip_direct_writable_types_.end(); | |
319 | } | |
20effc67 TL |
320 | void SetFilesystemActiveNoLock( |
321 | bool active, IOStatus error = IOStatus::Corruption("Not active")) { | |
322 | error.PermitUncheckedError(); | |
323 | filesystem_active_ = active; | |
324 | if (!active) { | |
325 | error_ = error; | |
326 | } | |
327 | } | |
328 | void SetFilesystemActive( | |
329 | bool active, IOStatus error = IOStatus::Corruption("Not active")) { | |
330 | MutexLock l(&mutex_); | |
331 | error.PermitUncheckedError(); | |
332 | SetFilesystemActiveNoLock(active, error); | |
333 | } | |
1e59de90 | 334 | void SetFilesystemDirectWritable(bool writable) { |
20effc67 TL |
335 | MutexLock l(&mutex_); |
336 | filesystem_writable_ = writable; | |
337 | } | |
1e59de90 | 338 | void AssertNoOpenFile() { assert(open_managed_files_.empty()); } |
20effc67 TL |
339 | |
340 | IOStatus GetError() { return error_; } | |
341 | ||
342 | void SetFileSystemIOError(IOStatus io_error) { | |
343 | MutexLock l(&mutex_); | |
344 | io_error.PermitUncheckedError(); | |
345 | error_ = io_error; | |
346 | } | |
347 | ||
1e59de90 TL |
348 | // To simulate the data corruption before data is written in FS |
349 | void IngestDataCorruptionBeforeWrite() { | |
350 | MutexLock l(&mutex_); | |
351 | ingest_data_corruption_before_write_ = true; | |
352 | } | |
353 | ||
354 | void NoDataCorruptionBeforeWrite() { | |
355 | MutexLock l(&mutex_); | |
356 | ingest_data_corruption_before_write_ = false; | |
357 | } | |
358 | ||
359 | bool ShouldDataCorruptionBeforeWrite() { | |
360 | MutexLock l(&mutex_); | |
361 | return ingest_data_corruption_before_write_; | |
362 | } | |
363 | ||
364 | void SetChecksumHandoffFuncType(const ChecksumType& func_type) { | |
365 | MutexLock l(&mutex_); | |
366 | checksum_handoff_func_tpye_ = func_type; | |
367 | } | |
368 | ||
369 | const ChecksumType& GetChecksumHandoffFuncType() { | |
370 | MutexLock l(&mutex_); | |
371 | return checksum_handoff_func_tpye_; | |
372 | } | |
373 | ||
374 | void SetFailGetUniqueId(bool flag) { | |
375 | MutexLock l(&mutex_); | |
376 | fail_get_file_unique_id_ = flag; | |
377 | } | |
378 | ||
379 | bool ShouldFailGetUniqueId() { | |
380 | MutexLock l(&mutex_); | |
381 | return fail_get_file_unique_id_; | |
382 | } | |
383 | ||
20effc67 TL |
384 | // Specify what the operation, so we can inject the right type of error |
385 | enum ErrorOperation : char { | |
386 | kRead = 0, | |
1e59de90 TL |
387 | kMultiReadSingleReq = 1, |
388 | kMultiRead = 2, | |
20effc67 TL |
389 | kOpen, |
390 | }; | |
391 | ||
392 | // Set thread-local parameters for error injection. The first argument, | |
393 | // seed is the seed for the random number generator, and one_in determines | |
394 | // the probability of injecting error (i.e an error is injected with | |
395 | // 1/one_in probability) | |
396 | void SetThreadLocalReadErrorContext(uint32_t seed, int one_in) { | |
397 | struct ErrorContext* ctx = | |
1e59de90 | 398 | static_cast<struct ErrorContext*>(thread_local_error_->Get()); |
20effc67 TL |
399 | if (ctx == nullptr) { |
400 | ctx = new ErrorContext(seed); | |
401 | thread_local_error_->Reset(ctx); | |
402 | } | |
403 | ctx->one_in = one_in; | |
404 | ctx->count = 0; | |
405 | } | |
406 | ||
1e59de90 | 407 | static void DeleteThreadLocalErrorContext(void* p) { |
20effc67 TL |
408 | ErrorContext* ctx = static_cast<ErrorContext*>(p); |
409 | delete ctx; | |
410 | } | |
411 | ||
1e59de90 TL |
412 | // This is to set the parameters for the write error injection. |
413 | // seed is the seed for the random number generator, and one_in determines | |
414 | // the probability of injecting error (i.e an error is injected with | |
415 | // 1/one_in probability). For write error, we can specify the error we | |
416 | // want to inject. Types decides the file types we want to inject the | |
417 | // error (e.g., Wal files, SST files), which is empty by default. | |
418 | void SetRandomWriteError(uint32_t seed, int one_in, IOStatus error, | |
419 | bool inject_for_all_file_types, | |
420 | const std::vector<FileType>& types) { | |
421 | MutexLock l(&mutex_); | |
422 | Random tmp_rand(seed); | |
423 | error.PermitUncheckedError(); | |
424 | error_ = error; | |
425 | write_error_rand_ = tmp_rand; | |
426 | write_error_one_in_ = one_in; | |
427 | inject_for_all_file_types_ = inject_for_all_file_types; | |
428 | write_error_allowed_types_ = types; | |
429 | } | |
430 | ||
431 | void SetSkipDirectWritableTypes(const std::set<FileType>& types) { | |
432 | MutexLock l(&mutex_); | |
433 | skip_direct_writable_types_ = types; | |
434 | } | |
435 | ||
436 | void SetRandomMetadataWriteError(int one_in) { | |
437 | MutexLock l(&mutex_); | |
438 | metadata_write_error_one_in_ = one_in; | |
439 | } | |
440 | // If the value is not 0, it is enabled. Otherwise, it is disabled. | |
441 | void SetRandomReadError(int one_in) { read_error_one_in_ = one_in; } | |
442 | ||
443 | bool ShouldInjectRandomReadError() { | |
444 | return read_error_one_in() && | |
445 | Random::GetTLSInstance()->OneIn(read_error_one_in()); | |
446 | } | |
447 | ||
448 | // Inject an write error with randomlized parameter and the predefined | |
449 | // error type. Only the allowed file types will inject the write error | |
450 | IOStatus InjectWriteError(const std::string& file_name); | |
451 | ||
452 | // Ingest error to metadata operations. | |
453 | IOStatus InjectMetadataWriteError(); | |
454 | ||
20effc67 TL |
455 | // Inject an error. For a READ operation, a status of IOError(), a |
456 | // corruption in the contents of scratch, or truncation of slice | |
457 | // are the types of error with equal probability. For OPEN, | |
458 | // its always an IOError. | |
1e59de90 TL |
459 | // fault_injected returns whether a fault is injected. It is needed |
460 | // because some fault is inected with IOStatus to be OK. | |
461 | IOStatus InjectThreadSpecificReadError(ErrorOperation op, Slice* slice, | |
462 | bool direct_io, char* scratch, | |
463 | bool need_count_increase, | |
464 | bool* fault_injected); | |
20effc67 TL |
465 | |
466 | // Get the count of how many times we injected since the previous call | |
467 | int GetAndResetErrorCount() { | |
1e59de90 | 468 | ErrorContext* ctx = static_cast<ErrorContext*>(thread_local_error_->Get()); |
20effc67 TL |
469 | int count = 0; |
470 | if (ctx != nullptr) { | |
471 | count = ctx->count; | |
472 | ctx->count = 0; | |
473 | } | |
474 | return count; | |
475 | } | |
476 | ||
477 | void EnableErrorInjection() { | |
1e59de90 | 478 | ErrorContext* ctx = static_cast<ErrorContext*>(thread_local_error_->Get()); |
20effc67 TL |
479 | if (ctx) { |
480 | ctx->enable_error_injection = true; | |
481 | } | |
482 | } | |
483 | ||
1e59de90 TL |
484 | void EnableWriteErrorInjection() { |
485 | MutexLock l(&mutex_); | |
486 | enable_write_error_injection_ = true; | |
487 | } | |
488 | void EnableMetadataWriteErrorInjection() { | |
489 | MutexLock l(&mutex_); | |
490 | enable_metadata_write_error_injection_ = true; | |
491 | } | |
492 | ||
493 | void DisableWriteErrorInjection() { | |
494 | MutexLock l(&mutex_); | |
495 | enable_write_error_injection_ = false; | |
496 | } | |
497 | ||
20effc67 | 498 | void DisableErrorInjection() { |
1e59de90 | 499 | ErrorContext* ctx = static_cast<ErrorContext*>(thread_local_error_->Get()); |
20effc67 TL |
500 | if (ctx) { |
501 | ctx->enable_error_injection = false; | |
502 | } | |
503 | } | |
504 | ||
1e59de90 TL |
505 | void DisableMetadataWriteErrorInjection() { |
506 | MutexLock l(&mutex_); | |
507 | enable_metadata_write_error_injection_ = false; | |
508 | } | |
509 | ||
510 | int read_error_one_in() const { return read_error_one_in_.load(); } | |
511 | ||
512 | int write_error_one_in() const { return write_error_one_in_; } | |
513 | ||
20effc67 TL |
514 | // We capture a backtrace every time a fault is injected, for debugging |
515 | // purposes. This call prints the backtrace to stderr and frees the | |
516 | // saved callstack | |
517 | void PrintFaultBacktrace(); | |
518 | ||
519 | private: | |
520 | port::Mutex mutex_; | |
521 | std::map<std::string, FSFileState> db_file_state_; | |
1e59de90 TL |
522 | std::set<std::string> open_managed_files_; |
523 | // directory -> (file name -> file contents to recover) | |
524 | // When data is recovered from unsyned parent directory, the files with | |
525 | // empty file contents to recover is deleted. Those with non-empty ones | |
526 | // will be recovered to content accordingly. | |
527 | std::unordered_map<std::string, std::map<std::string, std::string>> | |
20effc67 | 528 | dir_to_new_files_since_last_sync_; |
1e59de90 | 529 | bool filesystem_active_; // Record flushes, syncs, writes |
20effc67 TL |
530 | bool filesystem_writable_; // Bypass FaultInjectionTestFS and go directly |
531 | // to underlying FS for writable files | |
532 | IOStatus error_; | |
533 | ||
534 | enum ErrorType : int { | |
535 | kErrorTypeStatus = 0, | |
536 | kErrorTypeCorruption, | |
537 | kErrorTypeTruncated, | |
538 | kErrorTypeMax | |
539 | }; | |
540 | ||
541 | struct ErrorContext { | |
542 | Random rand; | |
543 | int one_in; | |
544 | int count; | |
545 | bool enable_error_injection; | |
546 | void* callstack; | |
1e59de90 | 547 | std::string message; |
20effc67 TL |
548 | int frames; |
549 | ErrorType type; | |
550 | ||
551 | explicit ErrorContext(uint32_t seed) | |
552 | : rand(seed), | |
553 | enable_error_injection(false), | |
554 | callstack(nullptr), | |
555 | frames(0) {} | |
556 | ~ErrorContext() { | |
557 | if (callstack) { | |
558 | free(callstack); | |
559 | } | |
560 | } | |
561 | }; | |
562 | ||
563 | std::unique_ptr<ThreadLocalPtr> thread_local_error_; | |
1e59de90 TL |
564 | bool enable_write_error_injection_; |
565 | bool enable_metadata_write_error_injection_; | |
566 | Random write_error_rand_; | |
567 | int write_error_one_in_; | |
568 | int metadata_write_error_one_in_; | |
569 | std::atomic<int> read_error_one_in_; | |
570 | bool inject_for_all_file_types_; | |
571 | std::vector<FileType> write_error_allowed_types_; | |
572 | // File types where direct writable is skipped. | |
573 | std::set<FileType> skip_direct_writable_types_; | |
574 | bool ingest_data_corruption_before_write_; | |
575 | ChecksumType checksum_handoff_func_tpye_; | |
576 | bool fail_get_file_unique_id_; | |
577 | ||
578 | // Extract number of type from file name. Return false if failing to fine | |
579 | // them. | |
580 | bool TryParseFileName(const std::string& file_name, uint64_t* number, | |
581 | FileType* type); | |
20effc67 TL |
582 | }; |
583 | ||
584 | } // namespace ROCKSDB_NAMESPACE |