ceph/src/rocksdb/utilities/fault_injection_fs.cc

   1 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
   2 //  This source code is licensed under both the GPLv2 (found in the
   3 //  COPYING file in the root directory) and Apache 2.0 License
   4 //  (found in the LICENSE.Apache file in the root directory).
   5 //
   6 // Copyright 2014 The LevelDB Authors. All rights reserved.
   7 // Use of this source code is governed by a BSD-style license that can be
   8 // found in the LICENSE file. See the AUTHORS file for names of contributors.
   9
  10 // This test uses a custom FileSystem to keep track of the state of a file
  11 // system the last "Sync". The data being written is cached in a "buffer".
  12 // Only when "Sync" is called, the data will be persistent. It can similate
  13 // file data loss (or entire files) not protected by a "Sync". For any of the
  14 // FileSystem related operations, by specify the "IOStatus Error", a specific
  15 // error can be returned when file system is not activated.
  16
  17 #include "utilities/fault_injection_fs.h"
  18
  19 #include <functional>
  20 #include <utility>
  21
  22 #include "env/composite_env_wrapper.h"
  23 #include "port/lang.h"
  24 #include "port/stack_trace.h"
  25 #include "util/random.h"
  26
  27 namespace ROCKSDB_NAMESPACE {
  28
  29 // Assume a filename, and not a directory name like "/foo/bar/"
  30 std::string TestFSGetDirName(const std::string filename) {
  31   size_t found = filename.find_last_of("/\\");
  32   if (found == std::string::npos) {
  33     return "";
  34   } else {
  35     return filename.substr(0, found);
  36   }
  37 }
  38
  39 // Trim the tailing "/" in the end of `str`
  40 std::string TestFSTrimDirname(const std::string& str) {
  41   size_t found = str.find_last_not_of("/");
  42   if (found == std::string::npos) {
  43     return str;
  44   }
  45   return str.substr(0, found + 1);
  46 }
  47
  48 // Return pair <parent directory name, file name> of a full path.
  49 std::pair<std::string, std::string> TestFSGetDirAndName(
  50     const std::string& name) {
  51   std::string dirname = TestFSGetDirName(name);
  52   std::string fname = name.substr(dirname.size() + 1);
  53   return std::make_pair(dirname, fname);
  54 }
  55
  56 IOStatus FSFileState::DropUnsyncedData() {
  57   buffer_.resize(0);
  58   return IOStatus::OK();
  59 }
  60
  61 IOStatus FSFileState::DropRandomUnsyncedData(Random* rand) {
  62   int range = static_cast<int>(buffer_.size());
  63   size_t truncated_size = static_cast<size_t>(rand->Uniform(range));
  64   buffer_.resize(truncated_size);
  65   return IOStatus::OK();
  66 }
  67
  68 IOStatus TestFSDirectory::Fsync(const IOOptions& options, IODebugContext* dbg) {
  69   if (!fs_->IsFilesystemActive()) {
  70     return fs_->GetError();
  71   }
  72   fs_->SyncDir(dirname_);
  73   return dir_->Fsync(options, dbg);
  74 }
  75
  76 TestFSWritableFile::TestFSWritableFile(const std::string& fname,
  77                                        std::unique_ptr<FSWritableFile>&& f,
  78                                        FaultInjectionTestFS* fs)
  79     : state_(fname),
  80       target_(std::move(f)),
  81       writable_file_opened_(true),
  82       fs_(fs) {
  83   assert(target_ != nullptr);
  84   state_.pos_ = 0;
  85 }
  86
  87 TestFSWritableFile::~TestFSWritableFile() {
  88   if (writable_file_opened_) {
  89     Close(IOOptions(), nullptr).PermitUncheckedError();
  90   }
  91 }
  92
  93 IOStatus TestFSWritableFile::Append(const Slice& data, const IOOptions&,
  94                                     IODebugContext*) {
  95   MutexLock l(&mutex_);
  96   if (!fs_->IsFilesystemActive()) {
  97     return fs_->GetError();
  98   }
  99   state_.buffer_.append(data.data(), data.size());
 100   state_.pos_ += data.size();
 101   fs_->WritableFileAppended(state_);
 102   return IOStatus::OK();
 103 }
 104
 105 IOStatus TestFSWritableFile::Close(const IOOptions& options,
 106                                    IODebugContext* dbg) {
 107   if (!fs_->IsFilesystemActive()) {
 108     return fs_->GetError();
 109   }
 110   writable_file_opened_ = false;
 111   IOStatus io_s;
 112   io_s = target_->Append(state_.buffer_, options, dbg);
 113   if (io_s.ok()) {
 114     state_.buffer_.resize(0);
 115     // Ignore sync errors
 116     target_->Sync(options, dbg).PermitUncheckedError();
 117     io_s = target_->Close(options, dbg);
 118   }
 119   if (io_s.ok()) {
 120     fs_->WritableFileClosed(state_);
 121   }
 122   return io_s;
 123 }
 124
 125 IOStatus TestFSWritableFile::Flush(const IOOptions&, IODebugContext*) {
 126   if (!fs_->IsFilesystemActive()) {
 127     return fs_->GetError();
 128   }
 129   if (fs_->IsFilesystemActive()) {
 130     state_.pos_at_last_flush_ = state_.pos_;
 131   }
 132   return IOStatus::OK();
 133 }
 134
 135 IOStatus TestFSWritableFile::Sync(const IOOptions& options,
 136                                   IODebugContext* dbg) {
 137   if (!fs_->IsFilesystemActive()) {
 138     return fs_->GetError();
 139   }
 140   IOStatus io_s = target_->Append(state_.buffer_, options, dbg);
 141   state_.buffer_.resize(0);
 142   // Ignore sync errors
 143   target_->Sync(options, dbg).PermitUncheckedError();
 144   state_.pos_at_last_sync_ = state_.pos_;
 145   fs_->WritableFileSynced(state_);
 146   return io_s;
 147 }
 148
 149 TestFSRandomRWFile::TestFSRandomRWFile(const std::string& /*fname*/,
 150                                        std::unique_ptr<FSRandomRWFile>&& f,
 151                                        FaultInjectionTestFS* fs)
 152     : target_(std::move(f)), file_opened_(true), fs_(fs) {
 153   assert(target_ != nullptr);
 154 }
 155
 156 TestFSRandomRWFile::~TestFSRandomRWFile() {
 157   if (file_opened_) {
 158     Close(IOOptions(), nullptr).PermitUncheckedError();
 159   }
 160 }
 161
 162 IOStatus TestFSRandomRWFile::Write(uint64_t offset, const Slice& data,
 163                                    const IOOptions& options,
 164                                    IODebugContext* dbg) {
 165   if (!fs_->IsFilesystemActive()) {
 166     return fs_->GetError();
 167   }
 168   return target_->Write(offset, data, options, dbg);
 169 }
 170
 171 IOStatus TestFSRandomRWFile::Read(uint64_t offset, size_t n,
 172                                   const IOOptions& options, Slice* result,
 173                                   char* scratch, IODebugContext* dbg) const {
 174   if (!fs_->IsFilesystemActive()) {
 175     return fs_->GetError();
 176   }
 177   return target_->Read(offset, n, options, result, scratch, dbg);
 178 }
 179
 180 IOStatus TestFSRandomRWFile::Close(const IOOptions& options,
 181                                    IODebugContext* dbg) {
 182   if (!fs_->IsFilesystemActive()) {
 183     return fs_->GetError();
 184   }
 185   file_opened_ = false;
 186   return target_->Close(options, dbg);
 187 }
 188
 189 IOStatus TestFSRandomRWFile::Flush(const IOOptions& options,
 190                                    IODebugContext* dbg) {
 191   if (!fs_->IsFilesystemActive()) {
 192     return fs_->GetError();
 193   }
 194   return target_->Flush(options, dbg);
 195 }
 196
 197 IOStatus TestFSRandomRWFile::Sync(const IOOptions& options,
 198                                   IODebugContext* dbg) {
 199   if (!fs_->IsFilesystemActive()) {
 200     return fs_->GetError();
 201   }
 202   return target_->Sync(options, dbg);
 203 }
 204
 205 TestFSRandomAccessFile::TestFSRandomAccessFile(const std::string& /*fname*/,
 206                                        std::unique_ptr<FSRandomAccessFile>&& f,
 207                                        FaultInjectionTestFS* fs)
 208     : target_(std::move(f)), fs_(fs) {
 209   assert(target_ != nullptr);
 210 }
 211
 212 IOStatus TestFSRandomAccessFile::Read(uint64_t offset, size_t n,
 213                                   const IOOptions& options, Slice* result,
 214                                   char* scratch, IODebugContext* dbg) const {
 215   if (!fs_->IsFilesystemActive()) {
 216     return fs_->GetError();
 217   }
 218   IOStatus s = target_->Read(offset, n, options, result, scratch, dbg);
 219   if (s.ok()) {
 220     s = fs_->InjectError(FaultInjectionTestFS::ErrorOperation::kRead, result,
 221                          use_direct_io(), scratch);
 222   }
 223   return s;
 224 }
 225
 226 IOStatus FaultInjectionTestFS::NewDirectory(
 227     const std::string& name, const IOOptions& options,
 228     std::unique_ptr<FSDirectory>* result, IODebugContext* dbg) {
 229   std::unique_ptr<FSDirectory> r;
 230   IOStatus io_s = target()->NewDirectory(name, options, &r, dbg);
 231   if (!io_s.ok()) {
 232     return io_s;
 233   }
 234   result->reset(
 235       new TestFSDirectory(this, TestFSTrimDirname(name), r.release()));
 236   return IOStatus::OK();
 237 }
 238
 239 IOStatus FaultInjectionTestFS::NewWritableFile(
 240     const std::string& fname, const FileOptions& file_opts,
 241     std::unique_ptr<FSWritableFile>* result, IODebugContext* dbg) {
 242   if (!IsFilesystemActive()) {
 243     return GetError();
 244   }
 245   if (IsFilesystemDirectWritable()) {
 246     return target()->NewWritableFile(fname, file_opts, result, dbg);
 247   }
 248
 249   IOStatus io_s = target()->NewWritableFile(fname, file_opts, result, dbg);
 250   if (io_s.ok()) {
 251     result->reset(new TestFSWritableFile(fname, std::move(*result), this));
 252     // WritableFileWriter* file is opened
 253     // again then it will be truncated - so forget our saved state.
 254     UntrackFile(fname);
 255     MutexLock l(&mutex_);
 256     open_files_.insert(fname);
 257     auto dir_and_name = TestFSGetDirAndName(fname);
 258     auto& list = dir_to_new_files_since_last_sync_[dir_and_name.first];
 259     list.insert(dir_and_name.second);
 260   }
 261   return io_s;
 262 }
 263
 264 IOStatus FaultInjectionTestFS::ReopenWritableFile(
 265     const std::string& fname, const FileOptions& file_opts,
 266     std::unique_ptr<FSWritableFile>* result, IODebugContext* dbg) {
 267   if (!IsFilesystemActive()) {
 268     return GetError();
 269   }
 270   if (IsFilesystemDirectWritable()) {
 271     return target()->ReopenWritableFile(fname, file_opts, result, dbg);
 272   }
 273   IOStatus io_s = target()->ReopenWritableFile(fname, file_opts, result, dbg);
 274   if (io_s.ok()) {
 275     result->reset(new TestFSWritableFile(fname, std::move(*result), this));
 276     // WritableFileWriter* file is opened
 277     // again then it will be truncated - so forget our saved state.
 278     UntrackFile(fname);
 279     MutexLock l(&mutex_);
 280     open_files_.insert(fname);
 281     auto dir_and_name = TestFSGetDirAndName(fname);
 282     auto& list = dir_to_new_files_since_last_sync_[dir_and_name.first];
 283     list.insert(dir_and_name.second);
 284   }
 285   return io_s;
 286 }
 287
 288 IOStatus FaultInjectionTestFS::NewRandomRWFile(
 289     const std::string& fname, const FileOptions& file_opts,
 290     std::unique_ptr<FSRandomRWFile>* result, IODebugContext* dbg) {
 291   if (!IsFilesystemActive()) {
 292     return GetError();
 293   }
 294   if (IsFilesystemDirectWritable()) {
 295     return target()->NewRandomRWFile(fname, file_opts, result, dbg);
 296   }
 297   IOStatus io_s = target()->NewRandomRWFile(fname, file_opts, result, dbg);
 298   if (io_s.ok()) {
 299     result->reset(new TestFSRandomRWFile(fname, std::move(*result), this));
 300     // WritableFileWriter* file is opened
 301     // again then it will be truncated - so forget our saved state.
 302     UntrackFile(fname);
 303     MutexLock l(&mutex_);
 304     open_files_.insert(fname);
 305     auto dir_and_name = TestFSGetDirAndName(fname);
 306     auto& list = dir_to_new_files_since_last_sync_[dir_and_name.first];
 307     list.insert(dir_and_name.second);
 308   }
 309   return io_s;
 310 }
 311
 312 IOStatus FaultInjectionTestFS::NewRandomAccessFile(
 313     const std::string& fname, const FileOptions& file_opts,
 314     std::unique_ptr<FSRandomAccessFile>* result, IODebugContext* dbg) {
 315   if (!IsFilesystemActive()) {
 316     return GetError();
 317   }
 318   IOStatus io_s = InjectError(ErrorOperation::kOpen, nullptr, false, nullptr);
 319   if (io_s.ok()) {
 320     io_s = target()->NewRandomAccessFile(fname, file_opts, result, dbg);
 321   }
 322   if (io_s.ok()) {
 323     result->reset(new TestFSRandomAccessFile(fname, std::move(*result), this));
 324   }
 325   return io_s;
 326 }
 327
 328 IOStatus FaultInjectionTestFS::DeleteFile(const std::string& f,
 329                                           const IOOptions& options,
 330                                           IODebugContext* dbg) {
 331   if (!IsFilesystemActive()) {
 332     return GetError();
 333   }
 334   IOStatus io_s = FileSystemWrapper::DeleteFile(f, options, dbg);
 335   if (io_s.ok()) {
 336     UntrackFile(f);
 337   }
 338   return io_s;
 339 }
 340
 341 IOStatus FaultInjectionTestFS::RenameFile(const std::string& s,
 342                                           const std::string& t,
 343                                           const IOOptions& options,
 344                                           IODebugContext* dbg) {
 345   if (!IsFilesystemActive()) {
 346     return GetError();
 347   }
 348   IOStatus io_s = FileSystemWrapper::RenameFile(s, t, options, dbg);
 349
 350   if (io_s.ok()) {
 351     MutexLock l(&mutex_);
 352     if (db_file_state_.find(s) != db_file_state_.end()) {
 353       db_file_state_[t] = db_file_state_[s];
 354       db_file_state_.erase(s);
 355     }
 356
 357     auto sdn = TestFSGetDirAndName(s);
 358     auto tdn = TestFSGetDirAndName(t);
 359     if (dir_to_new_files_since_last_sync_[sdn.first].erase(sdn.second) != 0) {
 360       auto& tlist = dir_to_new_files_since_last_sync_[tdn.first];
 361       assert(tlist.find(tdn.second) == tlist.end());
 362       tlist.insert(tdn.second);
 363     }
 364   }
 365
 366   return io_s;
 367 }
 368
 369 void FaultInjectionTestFS::WritableFileClosed(const FSFileState& state) {
 370   MutexLock l(&mutex_);
 371   if (open_files_.find(state.filename_) != open_files_.end()) {
 372     db_file_state_[state.filename_] = state;
 373     open_files_.erase(state.filename_);
 374   }
 375 }
 376
 377 void FaultInjectionTestFS::WritableFileSynced(const FSFileState& state) {
 378   MutexLock l(&mutex_);
 379   if (open_files_.find(state.filename_) != open_files_.end()) {
 380     if (db_file_state_.find(state.filename_) == db_file_state_.end()) {
 381       db_file_state_.insert(std::make_pair(state.filename_, state));
 382     } else {
 383       db_file_state_[state.filename_] = state;
 384     }
 385   }
 386 }
 387
 388 void FaultInjectionTestFS::WritableFileAppended(const FSFileState& state) {
 389   MutexLock l(&mutex_);
 390   if (open_files_.find(state.filename_) != open_files_.end()) {
 391     if (db_file_state_.find(state.filename_) == db_file_state_.end()) {
 392       db_file_state_.insert(std::make_pair(state.filename_, state));
 393     } else {
 394       db_file_state_[state.filename_] = state;
 395     }
 396   }
 397 }
 398
 399 IOStatus FaultInjectionTestFS::DropUnsyncedFileData() {
 400   IOStatus io_s;
 401   MutexLock l(&mutex_);
 402   for (std::map<std::string, FSFileState>::iterator it = db_file_state_.begin();
 403        io_s.ok() && it != db_file_state_.end(); ++it) {
 404     FSFileState& fs_state = it->second;
 405     if (!fs_state.IsFullySynced()) {
 406       io_s = fs_state.DropUnsyncedData();
 407     }
 408   }
 409   return io_s;
 410 }
 411
 412 IOStatus FaultInjectionTestFS::DropRandomUnsyncedFileData(Random* rnd) {
 413   IOStatus io_s;
 414   MutexLock l(&mutex_);
 415   for (std::map<std::string, FSFileState>::iterator it = db_file_state_.begin();
 416        io_s.ok() && it != db_file_state_.end(); ++it) {
 417     FSFileState& fs_state = it->second;
 418     if (!fs_state.IsFullySynced()) {
 419       io_s = fs_state.DropRandomUnsyncedData(rnd);
 420     }
 421   }
 422   return io_s;
 423 }
 424
 425 IOStatus FaultInjectionTestFS::DeleteFilesCreatedAfterLastDirSync(
 426     const IOOptions& options, IODebugContext* dbg) {
 427   // Because DeleteFile access this container make a copy to avoid deadlock
 428   std::map<std::string, std::set<std::string>> map_copy;
 429   {
 430     MutexLock l(&mutex_);
 431     map_copy.insert(dir_to_new_files_since_last_sync_.begin(),
 432                     dir_to_new_files_since_last_sync_.end());
 433   }
 434
 435   for (auto& pair : map_copy) {
 436     for (std::string name : pair.second) {
 437       IOStatus io_s = DeleteFile(pair.first + "/" + name, options, dbg);
 438       if (!io_s.ok()) {
 439         return io_s;
 440       }
 441     }
 442   }
 443   return IOStatus::OK();
 444 }
 445
 446 void FaultInjectionTestFS::ResetState() {
 447   MutexLock l(&mutex_);
 448   db_file_state_.clear();
 449   dir_to_new_files_since_last_sync_.clear();
 450   SetFilesystemActiveNoLock(true);
 451 }
 452
 453 void FaultInjectionTestFS::UntrackFile(const std::string& f) {
 454   MutexLock l(&mutex_);
 455   auto dir_and_name = TestFSGetDirAndName(f);
 456   dir_to_new_files_since_last_sync_[dir_and_name.first].erase(
 457       dir_and_name.second);
 458   db_file_state_.erase(f);
 459   open_files_.erase(f);
 460 }
 461
 462 IOStatus FaultInjectionTestFS::InjectError(ErrorOperation op,
 463                                            Slice* result,
 464                                            bool direct_io,
 465                                            char* scratch) {
 466   ErrorContext* ctx =
 467         static_cast<ErrorContext*>(thread_local_error_->Get());
 468   if (ctx == nullptr || !ctx->enable_error_injection || !ctx->one_in) {
 469     return IOStatus::OK();
 470   }
 471
 472   if (ctx->rand.OneIn(ctx->one_in)) {
 473     ctx->count++;
 474     if (ctx->callstack) {
 475       free(ctx->callstack);
 476     }
 477     ctx->callstack = port::SaveStack(&ctx->frames);
 478     switch (op) {
 479       case kRead:
 480       {
 481         if (!direct_io) {
 482           ctx->type =
 483             static_cast<ErrorType>(ctx->rand.Uniform(ErrorType::kErrorTypeMax));
 484         } else {
 485           // In Direct IO mode, the actual read will read extra data due to
 486           // alignment restrictions. So don't inject corruption or
 487           // truncated reads as we don't know if it will actually cause a
 488           // detectable error
 489           ctx->type = ErrorType::kErrorTypeStatus;
 490         }
 491         switch (ctx->type) {
 492           // Inject IO error
 493           case ErrorType::kErrorTypeStatus:
 494             return IOStatus::IOError();
 495           // Inject random corruption
 496           case ErrorType::kErrorTypeCorruption:
 497           {
 498             if (result->data() == scratch) {
 499               uint64_t offset = ctx->rand.Uniform((uint32_t)result->size());
 500               uint64_t len =
 501                 std::min<uint64_t>(result->size() - offset, 64UL);
 502               assert(offset < result->size());
 503               assert(offset + len <= result->size());
 504               std::string str;
 505               // The randomly generated string could be identical to the
 506               // original one, so retry
 507               do {
 508                 str = ctx->rand.RandomString(static_cast<int>(len));
 509               } while (str == std::string(scratch + offset, len));
 510               memcpy(scratch + offset, str.data(), len);
 511               break;
 512             } else {
 513               FALLTHROUGH_INTENDED;
 514             }
 515           }
 516           // Truncate the result
 517           case ErrorType::kErrorTypeTruncated:
 518           {
 519             assert(result->size() > 0);
 520             uint64_t offset = ctx->rand.Uniform((uint32_t)result->size());
 521             assert(offset < result->size());
 522             *result = Slice(result->data(), offset);
 523             break;
 524           }
 525           default:
 526             assert(false);
 527         }
 528         break;
 529       }
 530       case kOpen:
 531         return IOStatus::IOError();
 532       default:
 533         assert(false);
 534     }
 535   }
 536   return IOStatus::OK();
 537 }
 538
 539 void FaultInjectionTestFS::PrintFaultBacktrace() {
 540 #if defined(OS_LINUX)
 541   ErrorContext* ctx =
 542         static_cast<ErrorContext*>(thread_local_error_->Get());
 543   if (ctx == nullptr) {
 544     return;
 545   }
 546   fprintf(stderr, "Injected error type = %d\n", ctx->type);
 547   port::PrintAndFreeStack(ctx->callstack, ctx->frames);
 548   ctx->callstack = nullptr;
 549 #endif
 550 }
 551
 552 }  // namespace ROCKSDB_NAMESPACE