ceph/src/rocksdb/test_util/fault_injection_test_env.cc

   1 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
   2 //  This source code is licensed under both the GPLv2 (found in the
   3 //  COPYING file in the root directory) and Apache 2.0 License
   4 //  (found in the LICENSE.Apache file in the root directory).
   5 //
   6 // Copyright 2014 The LevelDB Authors. All rights reserved.
   7 // Use of this source code is governed by a BSD-style license that can be
   8 // found in the LICENSE file. See the AUTHORS file for names of contributors.
   9
  10 // This test uses a custom Env to keep track of the state of a filesystem as of
  11 // the last "sync". It then checks for data loss errors by purposely dropping
  12 // file data (or entire files) not protected by a "sync".
  13
  14 #include "test_util/fault_injection_test_env.h"
  15 #include <functional>
  16 #include <utility>
  17
  18 namespace ROCKSDB_NAMESPACE {
  19
  20 // Assume a filename, and not a directory name like "/foo/bar/"
  21 std::string GetDirName(const std::string filename) {
  22   size_t found = filename.find_last_of("/\\");
  23   if (found == std::string::npos) {
  24     return "";
  25   } else {
  26     return filename.substr(0, found);
  27   }
  28 }
  29
  30 // A basic file truncation function suitable for this test.
  31 Status Truncate(Env* env, const std::string& filename, uint64_t length) {
  32   std::unique_ptr<SequentialFile> orig_file;
  33   const EnvOptions options;
  34   Status s = env->NewSequentialFile(filename, &orig_file, options);
  35   if (!s.ok()) {
  36     fprintf(stderr, "Cannot open file %s for truncation: %s\n",
  37             filename.c_str(), s.ToString().c_str());
  38     return s;
  39   }
  40
  41   std::unique_ptr<char[]> scratch(new char[length]);
  42   ROCKSDB_NAMESPACE::Slice result;
  43   s = orig_file->Read(length, &result, scratch.get());
  44 #ifdef OS_WIN
  45   orig_file.reset();
  46 #endif
  47   if (s.ok()) {
  48     std::string tmp_name = GetDirName(filename) + "/truncate.tmp";
  49     std::unique_ptr<WritableFile> tmp_file;
  50     s = env->NewWritableFile(tmp_name, &tmp_file, options);
  51     if (s.ok()) {
  52       s = tmp_file->Append(result);
  53       if (s.ok()) {
  54         s = env->RenameFile(tmp_name, filename);
  55       } else {
  56         fprintf(stderr, "Cannot rename file %s to %s: %s\n", tmp_name.c_str(),
  57                 filename.c_str(), s.ToString().c_str());
  58         env->DeleteFile(tmp_name);
  59       }
  60     }
  61   }
  62   if (!s.ok()) {
  63     fprintf(stderr, "Cannot truncate file %s: %s\n", filename.c_str(),
  64             s.ToString().c_str());
  65   }
  66
  67   return s;
  68 }
  69
  70 // Trim the tailing "/" in the end of `str`
  71 std::string TrimDirname(const std::string& str) {
  72   size_t found = str.find_last_not_of("/");
  73   if (found == std::string::npos) {
  74     return str;
  75   }
  76   return str.substr(0, found + 1);
  77 }
  78
  79 // Return pair <parent directory name, file name> of a full path.
  80 std::pair<std::string, std::string> GetDirAndName(const std::string& name) {
  81   std::string dirname = GetDirName(name);
  82   std::string fname = name.substr(dirname.size() + 1);
  83   return std::make_pair(dirname, fname);
  84 }
  85
  86 Status FileState::DropUnsyncedData(Env* env) const {
  87   ssize_t sync_pos = pos_at_last_sync_ == -1 ? 0 : pos_at_last_sync_;
  88   return Truncate(env, filename_, sync_pos);
  89 }
  90
  91 Status FileState::DropRandomUnsyncedData(Env* env, Random* rand) const {
  92   ssize_t sync_pos = pos_at_last_sync_ == -1 ? 0 : pos_at_last_sync_;
  93   assert(pos_ >= sync_pos);
  94   int range = static_cast<int>(pos_ - sync_pos);
  95   uint64_t truncated_size =
  96       static_cast<uint64_t>(sync_pos) + rand->Uniform(range);
  97   return Truncate(env, filename_, truncated_size);
  98 }
  99
 100 Status TestDirectory::Fsync() {
 101   if (!env_->IsFilesystemActive()) {
 102     return env_->GetError();
 103   }
 104   env_->SyncDir(dirname_);
 105   return dir_->Fsync();
 106 }
 107
 108 TestWritableFile::TestWritableFile(const std::string& fname,
 109                                    std::unique_ptr<WritableFile>&& f,
 110                                    FaultInjectionTestEnv* env)
 111     : state_(fname),
 112       target_(std::move(f)),
 113       writable_file_opened_(true),
 114       env_(env) {
 115   assert(target_ != nullptr);
 116   state_.pos_ = 0;
 117 }
 118
 119 TestWritableFile::~TestWritableFile() {
 120   if (writable_file_opened_) {
 121     Close();
 122   }
 123 }
 124
 125 Status TestWritableFile::Append(const Slice& data) {
 126   if (!env_->IsFilesystemActive()) {
 127     return env_->GetError();
 128   }
 129   Status s = target_->Append(data);
 130   if (s.ok()) {
 131     state_.pos_ += data.size();
 132     env_->WritableFileAppended(state_);
 133   }
 134   return s;
 135 }
 136
 137 Status TestWritableFile::Close() {
 138   writable_file_opened_ = false;
 139   Status s = target_->Close();
 140   if (s.ok()) {
 141     env_->WritableFileClosed(state_);
 142   }
 143   return s;
 144 }
 145
 146 Status TestWritableFile::Flush() {
 147   Status s = target_->Flush();
 148   if (s.ok() && env_->IsFilesystemActive()) {
 149     state_.pos_at_last_flush_ = state_.pos_;
 150   }
 151   return s;
 152 }
 153
 154 Status TestWritableFile::Sync() {
 155   if (!env_->IsFilesystemActive()) {
 156     return Status::IOError("FaultInjectionTestEnv: not active");
 157   }
 158   // No need to actual sync.
 159   state_.pos_at_last_sync_ = state_.pos_;
 160   env_->WritableFileSynced(state_);
 161   return Status::OK();
 162 }
 163
 164 TestRandomRWFile::TestRandomRWFile(const std::string& /*fname*/,
 165                                    std::unique_ptr<RandomRWFile>&& f,
 166                                    FaultInjectionTestEnv* env)
 167     : target_(std::move(f)), file_opened_(true), env_(env) {
 168   assert(target_ != nullptr);
 169 }
 170
 171 TestRandomRWFile::~TestRandomRWFile() {
 172   if (file_opened_) {
 173     Close();
 174   }
 175 }
 176
 177 Status TestRandomRWFile::Write(uint64_t offset, const Slice& data) {
 178   if (!env_->IsFilesystemActive()) {
 179     return env_->GetError();
 180   }
 181   return target_->Write(offset, data);
 182 }
 183
 184 Status TestRandomRWFile::Read(uint64_t offset, size_t n, Slice* result,
 185                               char* scratch) const {
 186   if (!env_->IsFilesystemActive()) {
 187     return env_->GetError();
 188   }
 189   return target_->Read(offset, n, result, scratch);
 190 }
 191
 192 Status TestRandomRWFile::Close() {
 193   file_opened_ = false;
 194   return target_->Close();
 195 }
 196
 197 Status TestRandomRWFile::Flush() {
 198   if (!env_->IsFilesystemActive()) {
 199     return env_->GetError();
 200   }
 201   return target_->Flush();
 202 }
 203
 204 Status TestRandomRWFile::Sync() {
 205   if (!env_->IsFilesystemActive()) {
 206     return env_->GetError();
 207   }
 208   return target_->Sync();
 209 }
 210
 211 Status FaultInjectionTestEnv::NewDirectory(const std::string& name,
 212                                            std::unique_ptr<Directory>* result) {
 213   std::unique_ptr<Directory> r;
 214   Status s = target()->NewDirectory(name, &r);
 215   assert(s.ok());
 216   if (!s.ok()) {
 217     return s;
 218   }
 219   result->reset(new TestDirectory(this, TrimDirname(name), r.release()));
 220   return Status::OK();
 221 }
 222
 223 Status FaultInjectionTestEnv::NewWritableFile(
 224     const std::string& fname, std::unique_ptr<WritableFile>* result,
 225     const EnvOptions& soptions) {
 226   if (!IsFilesystemActive()) {
 227     return GetError();
 228   }
 229   // Not allow overwriting files
 230   Status s = target()->FileExists(fname);
 231   if (s.ok()) {
 232     return Status::Corruption("File already exists.");
 233   } else if (!s.IsNotFound()) {
 234     assert(s.IsIOError());
 235     return s;
 236   }
 237   s = target()->NewWritableFile(fname, result, soptions);
 238   if (s.ok()) {
 239     result->reset(new TestWritableFile(fname, std::move(*result), this));
 240     // WritableFileWriter* file is opened
 241     // again then it will be truncated - so forget our saved state.
 242     UntrackFile(fname);
 243     MutexLock l(&mutex_);
 244     open_files_.insert(fname);
 245     auto dir_and_name = GetDirAndName(fname);
 246     auto& list = dir_to_new_files_since_last_sync_[dir_and_name.first];
 247     list.insert(dir_and_name.second);
 248   }
 249   return s;
 250 }
 251
 252 Status FaultInjectionTestEnv::ReopenWritableFile(
 253     const std::string& fname, std::unique_ptr<WritableFile>* result,
 254     const EnvOptions& soptions) {
 255   if (!IsFilesystemActive()) {
 256     return GetError();
 257   }
 258   Status s = target()->ReopenWritableFile(fname, result, soptions);
 259   if (s.ok()) {
 260     result->reset(new TestWritableFile(fname, std::move(*result), this));
 261     // WritableFileWriter* file is opened
 262     // again then it will be truncated - so forget our saved state.
 263     UntrackFile(fname);
 264     MutexLock l(&mutex_);
 265     open_files_.insert(fname);
 266     auto dir_and_name = GetDirAndName(fname);
 267     auto& list = dir_to_new_files_since_last_sync_[dir_and_name.first];
 268     list.insert(dir_and_name.second);
 269   }
 270   return s;
 271 }
 272
 273 Status FaultInjectionTestEnv::NewRandomRWFile(
 274     const std::string& fname, std::unique_ptr<RandomRWFile>* result,
 275     const EnvOptions& soptions) {
 276   if (!IsFilesystemActive()) {
 277     return GetError();
 278   }
 279   Status s = target()->NewRandomRWFile(fname, result, soptions);
 280   if (s.ok()) {
 281     result->reset(new TestRandomRWFile(fname, std::move(*result), this));
 282     // WritableFileWriter* file is opened
 283     // again then it will be truncated - so forget our saved state.
 284     UntrackFile(fname);
 285     MutexLock l(&mutex_);
 286     open_files_.insert(fname);
 287     auto dir_and_name = GetDirAndName(fname);
 288     auto& list = dir_to_new_files_since_last_sync_[dir_and_name.first];
 289     list.insert(dir_and_name.second);
 290   }
 291   return s;
 292 }
 293
 294 Status FaultInjectionTestEnv::NewRandomAccessFile(
 295     const std::string& fname, std::unique_ptr<RandomAccessFile>* result,
 296     const EnvOptions& soptions) {
 297   if (!IsFilesystemActive()) {
 298     return GetError();
 299   }
 300   return target()->NewRandomAccessFile(fname, result, soptions);
 301 }
 302
 303 Status FaultInjectionTestEnv::DeleteFile(const std::string& f) {
 304   if (!IsFilesystemActive()) {
 305     return GetError();
 306   }
 307   Status s = EnvWrapper::DeleteFile(f);
 308   if (!s.ok()) {
 309     fprintf(stderr, "Cannot delete file %s: %s\n", f.c_str(),
 310             s.ToString().c_str());
 311   }
 312   if (s.ok()) {
 313     UntrackFile(f);
 314   }
 315   return s;
 316 }
 317
 318 Status FaultInjectionTestEnv::RenameFile(const std::string& s,
 319                                          const std::string& t) {
 320   if (!IsFilesystemActive()) {
 321     return GetError();
 322   }
 323   Status ret = EnvWrapper::RenameFile(s, t);
 324
 325   if (ret.ok()) {
 326     MutexLock l(&mutex_);
 327     if (db_file_state_.find(s) != db_file_state_.end()) {
 328       db_file_state_[t] = db_file_state_[s];
 329       db_file_state_.erase(s);
 330     }
 331
 332     auto sdn = GetDirAndName(s);
 333     auto tdn = GetDirAndName(t);
 334     if (dir_to_new_files_since_last_sync_[sdn.first].erase(sdn.second) != 0) {
 335       auto& tlist = dir_to_new_files_since_last_sync_[tdn.first];
 336       assert(tlist.find(tdn.second) == tlist.end());
 337       tlist.insert(tdn.second);
 338     }
 339   }
 340
 341   return ret;
 342 }
 343
 344 void FaultInjectionTestEnv::WritableFileClosed(const FileState& state) {
 345   MutexLock l(&mutex_);
 346   if (open_files_.find(state.filename_) != open_files_.end()) {
 347     db_file_state_[state.filename_] = state;
 348     open_files_.erase(state.filename_);
 349   }
 350 }
 351
 352 void FaultInjectionTestEnv::WritableFileSynced(const FileState& state) {
 353   MutexLock l(&mutex_);
 354   if (open_files_.find(state.filename_) != open_files_.end()) {
 355     if (db_file_state_.find(state.filename_) == db_file_state_.end()) {
 356       db_file_state_.insert(std::make_pair(state.filename_, state));
 357     } else {
 358       db_file_state_[state.filename_] = state;
 359     }
 360   }
 361 }
 362
 363 void FaultInjectionTestEnv::WritableFileAppended(const FileState& state) {
 364   MutexLock l(&mutex_);
 365   if (open_files_.find(state.filename_) != open_files_.end()) {
 366     if (db_file_state_.find(state.filename_) == db_file_state_.end()) {
 367       db_file_state_.insert(std::make_pair(state.filename_, state));
 368     } else {
 369       db_file_state_[state.filename_] = state;
 370     }
 371   }
 372 }
 373
 374 // For every file that is not fully synced, make a call to `func` with
 375 // FileState of the file as the parameter.
 376 Status FaultInjectionTestEnv::DropFileData(
 377     std::function<Status(Env*, FileState)> func) {
 378   Status s;
 379   MutexLock l(&mutex_);
 380   for (std::map<std::string, FileState>::const_iterator it =
 381            db_file_state_.begin();
 382        s.ok() && it != db_file_state_.end(); ++it) {
 383     const FileState& state = it->second;
 384     if (!state.IsFullySynced()) {
 385       s = func(target(), state);
 386     }
 387   }
 388   return s;
 389 }
 390
 391 Status FaultInjectionTestEnv::DropUnsyncedFileData() {
 392   return DropFileData([&](Env* env, const FileState& state) {
 393     return state.DropUnsyncedData(env);
 394   });
 395 }
 396
 397 Status FaultInjectionTestEnv::DropRandomUnsyncedFileData(Random* rnd) {
 398   return DropFileData([&](Env* env, const FileState& state) {
 399     return state.DropRandomUnsyncedData(env, rnd);
 400   });
 401 }
 402
 403 Status FaultInjectionTestEnv::DeleteFilesCreatedAfterLastDirSync() {
 404   // Because DeleteFile access this container make a copy to avoid deadlock
 405   std::map<std::string, std::set<std::string>> map_copy;
 406   {
 407     MutexLock l(&mutex_);
 408     map_copy.insert(dir_to_new_files_since_last_sync_.begin(),
 409                     dir_to_new_files_since_last_sync_.end());
 410   }
 411
 412   for (auto& pair : map_copy) {
 413     for (std::string name : pair.second) {
 414       Status s = DeleteFile(pair.first + "/" + name);
 415       if (!s.ok()) {
 416         return s;
 417       }
 418     }
 419   }
 420   return Status::OK();
 421 }
 422 void FaultInjectionTestEnv::ResetState() {
 423   MutexLock l(&mutex_);
 424   db_file_state_.clear();
 425   dir_to_new_files_since_last_sync_.clear();
 426   SetFilesystemActiveNoLock(true);
 427 }
 428
 429 void FaultInjectionTestEnv::UntrackFile(const std::string& f) {
 430   MutexLock l(&mutex_);
 431   auto dir_and_name = GetDirAndName(f);
 432   dir_to_new_files_since_last_sync_[dir_and_name.first].erase(
 433       dir_and_name.second);
 434   db_file_state_.erase(f);
 435   open_files_.erase(f);
 436 }
 437 }  // namespace ROCKSDB_NAMESPACE