ceph/src/rocksdb/util/fault_injection_test_env.cc

   1 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
   2 //  This source code is licensed under the BSD-style license found in the
   3 //  LICENSE file in the root directory of this source tree. An additional grant
   4 //  of patent rights can be found in the PATENTS file in the same directory.
   5 //
   6 // Copyright 2014 The LevelDB Authors. All rights reserved.
   7 // Use of this source code is governed by a BSD-style license that can be
   8 // found in the LICENSE file. See the AUTHORS file for names of contributors.
   9
  10 // This test uses a custom Env to keep track of the state of a filesystem as of
  11 // the last "sync". It then checks for data loss errors by purposely dropping
  12 // file data (or entire files) not protected by a "sync".
  13
  14 #include "util/fault_injection_test_env.h"
  15 #include <functional>
  16 #include <utility>
  17
  18 namespace rocksdb {
  19
  20 // Assume a filename, and not a directory name like "/foo/bar/"
  21 std::string GetDirName(const std::string filename) {
  22   size_t found = filename.find_last_of("/\\");
  23   if (found == std::string::npos) {
  24     return "";
  25   } else {
  26     return filename.substr(0, found);
  27   }
  28 }
  29
  30 // A basic file truncation function suitable for this test.
  31 Status Truncate(Env* env, const std::string& filename, uint64_t length) {
  32   unique_ptr<SequentialFile> orig_file;
  33   const EnvOptions options;
  34   Status s = env->NewSequentialFile(filename, &orig_file, options);
  35   if (!s.ok()) {
  36     fprintf(stderr, "Cannot truncate file %s: %s\n", filename.c_str(),
  37             s.ToString().c_str());
  38     return s;
  39   }
  40
  41   std::unique_ptr<char[]> scratch(new char[length]);
  42   rocksdb::Slice result;
  43   s = orig_file->Read(length, &result, scratch.get());
  44 #ifdef OS_WIN
  45   orig_file.reset();
  46 #endif
  47   if (s.ok()) {
  48     std::string tmp_name = GetDirName(filename) + "/truncate.tmp";
  49     unique_ptr<WritableFile> tmp_file;
  50     s = env->NewWritableFile(tmp_name, &tmp_file, options);
  51     if (s.ok()) {
  52       s = tmp_file->Append(result);
  53       if (s.ok()) {
  54         s = env->RenameFile(tmp_name, filename);
  55       } else {
  56         fprintf(stderr, "Cannot rename file %s to %s: %s\n", tmp_name.c_str(),
  57                 filename.c_str(), s.ToString().c_str());
  58         env->DeleteFile(tmp_name);
  59       }
  60     }
  61   }
  62   if (!s.ok()) {
  63     fprintf(stderr, "Cannot truncate file %s: %s\n", filename.c_str(),
  64             s.ToString().c_str());
  65   }
  66
  67   return s;
  68 }
  69
  70 // Trim the tailing "/" in the end of `str`
  71 std::string TrimDirname(const std::string& str) {
  72   size_t found = str.find_last_not_of("/");
  73   if (found == std::string::npos) {
  74     return str;
  75   }
  76   return str.substr(0, found + 1);
  77 }
  78
  79 // Return pair <parent directory name, file name> of a full path.
  80 std::pair<std::string, std::string> GetDirAndName(const std::string& name) {
  81   std::string dirname = GetDirName(name);
  82   std::string fname = name.substr(dirname.size() + 1);
  83   return std::make_pair(dirname, fname);
  84 }
  85
  86 Status FileState::DropUnsyncedData(Env* env) const {
  87   ssize_t sync_pos = pos_at_last_sync_ == -1 ? 0 : pos_at_last_sync_;
  88   return Truncate(env, filename_, sync_pos);
  89 }
  90
  91 Status FileState::DropRandomUnsyncedData(Env* env, Random* rand) const {
  92   ssize_t sync_pos = pos_at_last_sync_ == -1 ? 0 : pos_at_last_sync_;
  93   assert(pos_ >= sync_pos);
  94   int range = static_cast<int>(pos_ - sync_pos);
  95   uint64_t truncated_size =
  96       static_cast<uint64_t>(sync_pos) + rand->Uniform(range);
  97   return Truncate(env, filename_, truncated_size);
  98 }
  99
 100 Status TestDirectory::Fsync() {
 101   env_->SyncDir(dirname_);
 102   return dir_->Fsync();
 103 }
 104
 105 TestWritableFile::TestWritableFile(const std::string& fname,
 106                                    unique_ptr<WritableFile>&& f,
 107                                    FaultInjectionTestEnv* env)
 108     : state_(fname),
 109       target_(std::move(f)),
 110       writable_file_opened_(true),
 111       env_(env) {
 112   assert(target_ != nullptr);
 113   state_.pos_ = 0;
 114 }
 115
 116 TestWritableFile::~TestWritableFile() {
 117   if (writable_file_opened_) {
 118     Close();
 119   }
 120 }
 121
 122 Status TestWritableFile::Append(const Slice& data) {
 123   if (!env_->IsFilesystemActive()) {
 124     return Status::Corruption("Not Active");
 125   }
 126   Status s = target_->Append(data);
 127   if (s.ok()) {
 128     state_.pos_ += data.size();
 129   }
 130   return s;
 131 }
 132
 133 Status TestWritableFile::Close() {
 134   writable_file_opened_ = false;
 135   Status s = target_->Close();
 136   if (s.ok()) {
 137     env_->WritableFileClosed(state_);
 138   }
 139   return s;
 140 }
 141
 142 Status TestWritableFile::Flush() {
 143   Status s = target_->Flush();
 144   if (s.ok() && env_->IsFilesystemActive()) {
 145     state_.pos_at_last_flush_ = state_.pos_;
 146   }
 147   return s;
 148 }
 149
 150 Status TestWritableFile::Sync() {
 151   if (!env_->IsFilesystemActive()) {
 152     return Status::IOError("FaultInjectionTestEnv: not active");
 153   }
 154   // No need to actual sync.
 155   state_.pos_at_last_sync_ = state_.pos_;
 156   return Status::OK();
 157 }
 158
 159 Status FaultInjectionTestEnv::NewDirectory(const std::string& name,
 160                                            unique_ptr<Directory>* result) {
 161   unique_ptr<Directory> r;
 162   Status s = target()->NewDirectory(name, &r);
 163   assert(s.ok());
 164   if (!s.ok()) {
 165     return s;
 166   }
 167   result->reset(new TestDirectory(this, TrimDirname(name), r.release()));
 168   return Status::OK();
 169 }
 170
 171 Status FaultInjectionTestEnv::NewWritableFile(const std::string& fname,
 172                                               unique_ptr<WritableFile>* result,
 173                                               const EnvOptions& soptions) {
 174   if (!IsFilesystemActive()) {
 175     return Status::Corruption("Not Active");
 176   }
 177   // Not allow overwriting files
 178   Status s = target()->FileExists(fname);
 179   if (s.ok()) {
 180     return Status::Corruption("File already exists.");
 181   } else if (!s.IsNotFound()) {
 182     assert(s.IsIOError());
 183     return s;
 184   }
 185   s = target()->NewWritableFile(fname, result, soptions);
 186   if (s.ok()) {
 187     result->reset(new TestWritableFile(fname, std::move(*result), this));
 188     // WritableFileWriter* file is opened
 189     // again then it will be truncated - so forget our saved state.
 190     UntrackFile(fname);
 191     MutexLock l(&mutex_);
 192     open_files_.insert(fname);
 193     auto dir_and_name = GetDirAndName(fname);
 194     auto& list = dir_to_new_files_since_last_sync_[dir_and_name.first];
 195     list.insert(dir_and_name.second);
 196   }
 197   return s;
 198 }
 199
 200 Status FaultInjectionTestEnv::DeleteFile(const std::string& f) {
 201   if (!IsFilesystemActive()) {
 202     return Status::Corruption("Not Active");
 203   }
 204   Status s = EnvWrapper::DeleteFile(f);
 205   if (!s.ok()) {
 206     fprintf(stderr, "Cannot delete file %s: %s\n", f.c_str(),
 207             s.ToString().c_str());
 208   }
 209   assert(s.ok());
 210   if (s.ok()) {
 211     UntrackFile(f);
 212   }
 213   return s;
 214 }
 215
 216 Status FaultInjectionTestEnv::RenameFile(const std::string& s,
 217                                          const std::string& t) {
 218   if (!IsFilesystemActive()) {
 219     return Status::Corruption("Not Active");
 220   }
 221   Status ret = EnvWrapper::RenameFile(s, t);
 222
 223   if (ret.ok()) {
 224     MutexLock l(&mutex_);
 225     if (db_file_state_.find(s) != db_file_state_.end()) {
 226       db_file_state_[t] = db_file_state_[s];
 227       db_file_state_.erase(s);
 228     }
 229
 230     auto sdn = GetDirAndName(s);
 231     auto tdn = GetDirAndName(t);
 232     if (dir_to_new_files_since_last_sync_[sdn.first].erase(sdn.second) != 0) {
 233       auto& tlist = dir_to_new_files_since_last_sync_[tdn.first];
 234       assert(tlist.find(tdn.second) == tlist.end());
 235       tlist.insert(tdn.second);
 236     }
 237   }
 238
 239   return ret;
 240 }
 241
 242 void FaultInjectionTestEnv::WritableFileClosed(const FileState& state) {
 243   MutexLock l(&mutex_);
 244   if (open_files_.find(state.filename_) != open_files_.end()) {
 245     db_file_state_[state.filename_] = state;
 246     open_files_.erase(state.filename_);
 247   }
 248 }
 249
 250 // For every file that is not fully synced, make a call to `func` with
 251 // FileState of the file as the parameter.
 252 Status FaultInjectionTestEnv::DropFileData(
 253     std::function<Status(Env*, FileState)> func) {
 254   Status s;
 255   MutexLock l(&mutex_);
 256   for (std::map<std::string, FileState>::const_iterator it =
 257            db_file_state_.begin();
 258        s.ok() && it != db_file_state_.end(); ++it) {
 259     const FileState& state = it->second;
 260     if (!state.IsFullySynced()) {
 261       s = func(target(), state);
 262     }
 263   }
 264   return s;
 265 }
 266
 267 Status FaultInjectionTestEnv::DropUnsyncedFileData() {
 268   return DropFileData([&](Env* env, const FileState& state) {
 269     return state.DropUnsyncedData(env);
 270   });
 271 }
 272
 273 Status FaultInjectionTestEnv::DropRandomUnsyncedFileData(Random* rnd) {
 274   return DropFileData([&](Env* env, const FileState& state) {
 275     return state.DropRandomUnsyncedData(env, rnd);
 276   });
 277 }
 278
 279 Status FaultInjectionTestEnv::DeleteFilesCreatedAfterLastDirSync() {
 280   // Because DeleteFile access this container make a copy to avoid deadlock
 281   std::map<std::string, std::set<std::string>> map_copy;
 282   {
 283     MutexLock l(&mutex_);
 284     map_copy.insert(dir_to_new_files_since_last_sync_.begin(),
 285                     dir_to_new_files_since_last_sync_.end());
 286   }
 287
 288   for (auto& pair : map_copy) {
 289     for (std::string name : pair.second) {
 290       Status s = DeleteFile(pair.first + "/" + name);
 291       if (!s.ok()) {
 292         return s;
 293       }
 294     }
 295   }
 296   return Status::OK();
 297 }
 298 void FaultInjectionTestEnv::ResetState() {
 299   MutexLock l(&mutex_);
 300   db_file_state_.clear();
 301   dir_to_new_files_since_last_sync_.clear();
 302   SetFilesystemActiveNoLock(true);
 303 }
 304
 305 void FaultInjectionTestEnv::UntrackFile(const std::string& f) {
 306   MutexLock l(&mutex_);
 307   auto dir_and_name = GetDirAndName(f);
 308   dir_to_new_files_since_last_sync_[dir_and_name.first].erase(
 309       dir_and_name.second);
 310   db_file_state_.erase(f);
 311   open_files_.erase(f);
 312 }
 313 }  // namespace rocksdb