ceph/src/rocksdb/env/mock_env.cc

   1 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
   2 //  This source code is licensed under both the GPLv2 (found in the
   3 //  COPYING file in the root directory) and Apache 2.0 License
   4 //  (found in the LICENSE.Apache file in the root directory).
   5 //
   6 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
   7 // Use of this source code is governed by a BSD-style license that can be
   8 // found in the LICENSE file. See the AUTHORS file for names of contributors.
   9
  10 #include "env/mock_env.h"
  11
  12 #include <algorithm>
  13 #include <chrono>
  14
  15 #include "env/emulated_clock.h"
  16 #include "file/filename.h"
  17 #include "port/sys_time.h"
  18 #include "rocksdb/file_system.h"
  19 #include "rocksdb/utilities/options_type.h"
  20 #include "test_util/sync_point.h"
  21 #include "util/cast_util.h"
  22 #include "util/hash.h"
  23 #include "util/random.h"
  24 #include "util/rate_limiter.h"
  25 #include "util/string_util.h"
  26
  27 namespace ROCKSDB_NAMESPACE {
  28 namespace {
  29 int64_t MaybeCurrentTime(const std::shared_ptr<SystemClock>& clock) {
  30   int64_t time = 1337346000;  // arbitrary fallback default
  31   clock->GetCurrentTime(&time).PermitUncheckedError();
  32   return time;
  33 }
  34
  35 static std::unordered_map<std::string, OptionTypeInfo> time_elapse_type_info = {
  36 #ifndef ROCKSDB_LITE
  37     {"time_elapse_only_sleep",
  38      {0, OptionType::kBoolean, OptionVerificationType::kNormal,
  39       OptionTypeFlags::kCompareNever,
  40       [](const ConfigOptions& /*opts*/, const std::string& /*name*/,
  41          const std::string& value, void* addr) {
  42         auto clock = static_cast<EmulatedSystemClock*>(addr);
  43         clock->SetTimeElapseOnlySleep(ParseBoolean("", value));
  44         return Status::OK();
  45       },
  46       [](const ConfigOptions& /*opts*/, const std::string& /*name*/,
  47          const void* addr, std::string* value) {
  48         const auto clock = static_cast<const EmulatedSystemClock*>(addr);
  49         *value = clock->IsTimeElapseOnlySleep() ? "true" : "false";
  50         return Status::OK();
  51       },
  52       nullptr}},
  53 #endif  // ROCKSDB_LITE
  54 };
  55 static std::unordered_map<std::string, OptionTypeInfo> mock_sleep_type_info = {
  56 #ifndef ROCKSDB_LITE
  57     {"mock_sleep",
  58      {0, OptionType::kBoolean, OptionVerificationType::kNormal,
  59       OptionTypeFlags::kCompareNever,
  60       [](const ConfigOptions& /*opts*/, const std::string& /*name*/,
  61          const std::string& value, void* addr) {
  62         auto clock = static_cast<EmulatedSystemClock*>(addr);
  63         clock->SetMockSleep(ParseBoolean("", value));
  64         return Status::OK();
  65       },
  66       [](const ConfigOptions& /*opts*/, const std::string& /*name*/,
  67          const void* addr, std::string* value) {
  68         const auto clock = static_cast<const EmulatedSystemClock*>(addr);
  69         *value = clock->IsMockSleepEnabled() ? "true" : "false";
  70         return Status::OK();
  71       },
  72       nullptr}},
  73 #endif  // ROCKSDB_LITE
  74 };
  75 }  // namespace
  76
  77 EmulatedSystemClock::EmulatedSystemClock(
  78     const std::shared_ptr<SystemClock>& base, bool time_elapse_only_sleep)
  79     : SystemClockWrapper(base),
  80       maybe_starting_time_(MaybeCurrentTime(base)),
  81       time_elapse_only_sleep_(time_elapse_only_sleep),
  82       no_slowdown_(time_elapse_only_sleep) {
  83   RegisterOptions("", this, &time_elapse_type_info);
  84   RegisterOptions("", this, &mock_sleep_type_info);
  85 }
  86
  87 class MemFile {
  88  public:
  89   explicit MemFile(SystemClock* clock, const std::string& fn,
  90                    bool _is_lock_file = false)
  91       : clock_(clock),
  92         fn_(fn),
  93         refs_(0),
  94         is_lock_file_(_is_lock_file),
  95         locked_(false),
  96         size_(0),
  97         modified_time_(Now()),
  98         rnd_(Lower32of64(GetSliceNPHash64(fn))),
  99         fsynced_bytes_(0) {}
 100   // No copying allowed.
 101   MemFile(const MemFile&) = delete;
 102   void operator=(const MemFile&) = delete;
 103
 104   void Ref() {
 105     MutexLock lock(&mutex_);
 106     ++refs_;
 107   }
 108
 109   bool is_lock_file() const { return is_lock_file_; }
 110
 111   bool Lock() {
 112     assert(is_lock_file_);
 113     MutexLock lock(&mutex_);
 114     if (locked_) {
 115       return false;
 116     } else {
 117       locked_ = true;
 118       return true;
 119     }
 120   }
 121
 122   void Unlock() {
 123     assert(is_lock_file_);
 124     MutexLock lock(&mutex_);
 125     locked_ = false;
 126   }
 127
 128   void Unref() {
 129     bool do_delete = false;
 130     {
 131       MutexLock lock(&mutex_);
 132       --refs_;
 133       assert(refs_ >= 0);
 134       if (refs_ <= 0) {
 135         do_delete = true;
 136       }
 137     }
 138
 139     if (do_delete) {
 140       delete this;
 141     }
 142   }
 143
 144   uint64_t Size() const { return size_; }
 145
 146   void Truncate(size_t size, const IOOptions& /*options*/,
 147                 IODebugContext* /*dbg*/) {
 148     MutexLock lock(&mutex_);
 149     if (size < size_) {
 150       data_.resize(size);
 151       size_ = size;
 152     }
 153   }
 154
 155   void CorruptBuffer() {
 156     if (fsynced_bytes_ >= size_) {
 157       return;
 158     }
 159     uint64_t buffered_bytes = size_ - fsynced_bytes_;
 160     uint64_t start =
 161         fsynced_bytes_ + rnd_.Uniform(static_cast<int>(buffered_bytes));
 162     uint64_t end = std::min(start + 512, size_.load());
 163     MutexLock lock(&mutex_);
 164     for (uint64_t pos = start; pos < end; ++pos) {
 165       data_[static_cast<size_t>(pos)] = static_cast<char>(rnd_.Uniform(256));
 166     }
 167   }
 168
 169   IOStatus Read(uint64_t offset, size_t n, const IOOptions& /*options*/,
 170                 Slice* result, char* scratch, IODebugContext* /*dbg*/) const {
 171     {
 172       IOStatus s;
 173       TEST_SYNC_POINT_CALLBACK("MemFile::Read:IOStatus", &s);
 174       if (!s.ok()) {
 175         // with sync point only
 176         *result = Slice();
 177         return s;
 178       }
 179     }
 180     MutexLock lock(&mutex_);
 181     const uint64_t available = Size() - std::min(Size(), offset);
 182     size_t offset_ = static_cast<size_t>(offset);
 183     if (n > available) {
 184       n = static_cast<size_t>(available);
 185     }
 186     if (n == 0) {
 187       *result = Slice();
 188       return IOStatus::OK();
 189     }
 190     if (scratch) {
 191       memcpy(scratch, &(data_[offset_]), n);
 192       *result = Slice(scratch, n);
 193     } else {
 194       *result = Slice(&(data_[offset_]), n);
 195     }
 196     return IOStatus::OK();
 197   }
 198
 199   IOStatus Write(uint64_t offset, const Slice& data,
 200                  const IOOptions& /*options*/, IODebugContext* /*dbg*/) {
 201     MutexLock lock(&mutex_);
 202     size_t offset_ = static_cast<size_t>(offset);
 203     if (offset + data.size() > data_.size()) {
 204       data_.resize(offset_ + data.size());
 205     }
 206     data_.replace(offset_, data.size(), data.data(), data.size());
 207     size_ = data_.size();
 208     modified_time_ = Now();
 209     return IOStatus::OK();
 210   }
 211
 212   IOStatus Append(const Slice& data, const IOOptions& /*options*/,
 213                   IODebugContext* /*dbg*/) {
 214     MutexLock lock(&mutex_);
 215     data_.append(data.data(), data.size());
 216     size_ = data_.size();
 217     modified_time_ = Now();
 218     return IOStatus::OK();
 219   }
 220
 221   IOStatus Fsync(const IOOptions& /*options*/, IODebugContext* /*dbg*/) {
 222     fsynced_bytes_ = size_.load();
 223     return IOStatus::OK();
 224   }
 225
 226   uint64_t ModifiedTime() const { return modified_time_; }
 227
 228  private:
 229   uint64_t Now() {
 230     int64_t unix_time = 0;
 231     auto s = clock_->GetCurrentTime(&unix_time);
 232     assert(s.ok());
 233     return static_cast<uint64_t>(unix_time);
 234   }
 235
 236   // Private since only Unref() should be used to delete it.
 237   ~MemFile() { assert(refs_ == 0); }
 238
 239   SystemClock* clock_;
 240   const std::string fn_;
 241   mutable port::Mutex mutex_;
 242   int refs_;
 243   bool is_lock_file_;
 244   bool locked_;
 245
 246   // Data written into this file, all bytes before fsynced_bytes are
 247   // persistent.
 248   std::string data_;
 249   std::atomic<uint64_t> size_;
 250   std::atomic<uint64_t> modified_time_;
 251
 252   Random rnd_;
 253   std::atomic<uint64_t> fsynced_bytes_;
 254 };
 255
 256 namespace {
 257
 258 class MockSequentialFile : public FSSequentialFile {
 259  public:
 260   explicit MockSequentialFile(MemFile* file, const FileOptions& opts)
 261       : file_(file),
 262         use_direct_io_(opts.use_direct_reads),
 263         use_mmap_read_(opts.use_mmap_reads),
 264         pos_(0) {
 265     file_->Ref();
 266   }
 267
 268   ~MockSequentialFile() override { file_->Unref(); }
 269
 270   IOStatus Read(size_t n, const IOOptions& options, Slice* result,
 271                 char* scratch, IODebugContext* dbg) override {
 272     IOStatus s = file_->Read(pos_, n, options, result,
 273                              (use_mmap_read_) ? nullptr : scratch, dbg);
 274     if (s.ok()) {
 275       pos_ += result->size();
 276     }
 277     return s;
 278   }
 279
 280   bool use_direct_io() const override { return use_direct_io_; }
 281   IOStatus Skip(uint64_t n) override {
 282     if (pos_ > file_->Size()) {
 283       return IOStatus::IOError("pos_ > file_->Size()");
 284     }
 285     const uint64_t available = file_->Size() - pos_;
 286     if (n > available) {
 287       n = available;
 288     }
 289     pos_ += static_cast<size_t>(n);
 290     return IOStatus::OK();
 291   }
 292
 293  private:
 294   MemFile* file_;
 295   bool use_direct_io_;
 296   bool use_mmap_read_;
 297   size_t pos_;
 298 };
 299
 300 class MockRandomAccessFile : public FSRandomAccessFile {
 301  public:
 302   explicit MockRandomAccessFile(MemFile* file, const FileOptions& opts)
 303       : file_(file),
 304         use_direct_io_(opts.use_direct_reads),
 305         use_mmap_read_(opts.use_mmap_reads) {
 306     file_->Ref();
 307   }
 308
 309   ~MockRandomAccessFile() override { file_->Unref(); }
 310
 311   bool use_direct_io() const override { return use_direct_io_; }
 312
 313   IOStatus Prefetch(uint64_t /*offset*/, size_t /*n*/,
 314                     const IOOptions& /*options*/,
 315                     IODebugContext* /*dbg*/) override {
 316     return IOStatus::OK();
 317   }
 318
 319   IOStatus Read(uint64_t offset, size_t n, const IOOptions& options,
 320                 Slice* result, char* scratch,
 321                 IODebugContext* dbg) const override {
 322     if (use_mmap_read_) {
 323       return file_->Read(offset, n, options, result, nullptr, dbg);
 324     } else {
 325       return file_->Read(offset, n, options, result, scratch, dbg);
 326     }
 327   }
 328
 329  private:
 330   MemFile* file_;
 331   bool use_direct_io_;
 332   bool use_mmap_read_;
 333 };
 334
 335 class MockRandomRWFile : public FSRandomRWFile {
 336  public:
 337   explicit MockRandomRWFile(MemFile* file) : file_(file) { file_->Ref(); }
 338
 339   ~MockRandomRWFile() override { file_->Unref(); }
 340
 341   IOStatus Write(uint64_t offset, const Slice& data, const IOOptions& options,
 342                  IODebugContext* dbg) override {
 343     return file_->Write(offset, data, options, dbg);
 344   }
 345
 346   IOStatus Read(uint64_t offset, size_t n, const IOOptions& options,
 347                 Slice* result, char* scratch,
 348                 IODebugContext* dbg) const override {
 349     return file_->Read(offset, n, options, result, scratch, dbg);
 350   }
 351
 352   IOStatus Close(const IOOptions& options, IODebugContext* dbg) override {
 353     return file_->Fsync(options, dbg);
 354   }
 355
 356   IOStatus Flush(const IOOptions& /*options*/,
 357                  IODebugContext* /*dbg*/) override {
 358     return IOStatus::OK();
 359   }
 360
 361   IOStatus Sync(const IOOptions& options, IODebugContext* dbg) override {
 362     return file_->Fsync(options, dbg);
 363   }
 364
 365  private:
 366   MemFile* file_;
 367 };
 368
 369 class MockWritableFile : public FSWritableFile {
 370  public:
 371   MockWritableFile(MemFile* file, const FileOptions& opts)
 372       : file_(file),
 373         use_direct_io_(opts.use_direct_writes),
 374         rate_limiter_(opts.rate_limiter) {
 375     file_->Ref();
 376   }
 377
 378   ~MockWritableFile() override { file_->Unref(); }
 379
 380   bool use_direct_io() const override { return false && use_direct_io_; }
 381
 382   using FSWritableFile::Append;
 383   IOStatus Append(const Slice& data, const IOOptions& options,
 384                   IODebugContext* dbg) override {
 385     size_t bytes_written = 0;
 386     while (bytes_written < data.size()) {
 387       auto bytes = RequestToken(data.size() - bytes_written);
 388       IOStatus s = file_->Append(Slice(data.data() + bytes_written, bytes),
 389                                  options, dbg);
 390       if (!s.ok()) {
 391         return s;
 392       }
 393       bytes_written += bytes;
 394     }
 395     return IOStatus::OK();
 396   }
 397
 398   using FSWritableFile::PositionedAppend;
 399   IOStatus PositionedAppend(const Slice& data, uint64_t /*offset*/,
 400                             const IOOptions& options,
 401                             IODebugContext* dbg) override {
 402     assert(use_direct_io_);
 403     return Append(data, options, dbg);
 404   }
 405
 406   IOStatus Truncate(uint64_t size, const IOOptions& options,
 407                     IODebugContext* dbg) override {
 408     file_->Truncate(static_cast<size_t>(size), options, dbg);
 409     return IOStatus::OK();
 410   }
 411   IOStatus Close(const IOOptions& options, IODebugContext* dbg) override {
 412     return file_->Fsync(options, dbg);
 413   }
 414
 415   IOStatus Flush(const IOOptions& /*options*/,
 416                  IODebugContext* /*dbg*/) override {
 417     return IOStatus::OK();
 418   }
 419
 420   IOStatus Sync(const IOOptions& options, IODebugContext* dbg) override {
 421     return file_->Fsync(options, dbg);
 422   }
 423
 424   uint64_t GetFileSize(const IOOptions& /*options*/,
 425                        IODebugContext* /*dbg*/) override {
 426     return file_->Size();
 427   }
 428
 429  private:
 430   inline size_t RequestToken(size_t bytes) {
 431     if (rate_limiter_ && io_priority_ < Env::IO_TOTAL) {
 432       bytes = std::min(
 433           bytes, static_cast<size_t>(rate_limiter_->GetSingleBurstBytes()));
 434       rate_limiter_->Request(bytes, io_priority_);
 435     }
 436     return bytes;
 437   }
 438
 439   MemFile* file_;
 440   bool use_direct_io_;
 441   RateLimiter* rate_limiter_;
 442 };
 443
 444 class MockEnvDirectory : public FSDirectory {
 445  public:
 446   IOStatus Fsync(const IOOptions& /*options*/,
 447                  IODebugContext* /*dbg*/) override {
 448     return IOStatus::OK();
 449   }
 450
 451   IOStatus Close(const IOOptions& /*options*/,
 452                  IODebugContext* /*dbg*/) override {
 453     return IOStatus::OK();
 454   }
 455 };
 456
 457 class MockEnvFileLock : public FileLock {
 458  public:
 459   explicit MockEnvFileLock(const std::string& fname) : fname_(fname) {}
 460
 461   std::string FileName() const { return fname_; }
 462
 463  private:
 464   const std::string fname_;
 465 };
 466
 467 class TestMemLogger : public Logger {
 468  private:
 469   std::unique_ptr<FSWritableFile> file_;
 470   std::atomic_size_t log_size_;
 471   static const uint64_t flush_every_seconds_ = 5;
 472   std::atomic_uint_fast64_t last_flush_micros_;
 473   SystemClock* clock_;
 474   IOOptions options_;
 475   IODebugContext* dbg_;
 476   std::atomic<bool> flush_pending_;
 477
 478  public:
 479   TestMemLogger(std::unique_ptr<FSWritableFile> f, SystemClock* clock,
 480                 const IOOptions& options, IODebugContext* dbg,
 481                 const InfoLogLevel log_level = InfoLogLevel::ERROR_LEVEL)
 482       : Logger(log_level),
 483         file_(std::move(f)),
 484         log_size_(0),
 485         last_flush_micros_(0),
 486         clock_(clock),
 487         options_(options),
 488         dbg_(dbg),
 489         flush_pending_(false) {}
 490   ~TestMemLogger() override {}
 491
 492   void Flush() override {
 493     if (flush_pending_) {
 494       flush_pending_ = false;
 495     }
 496     last_flush_micros_ = clock_->NowMicros();
 497   }
 498
 499   using Logger::Logv;
 500   void Logv(const char* format, va_list ap) override {
 501     // We try twice: the first time with a fixed-size stack allocated buffer,
 502     // and the second time with a much larger dynamically allocated buffer.
 503     char buffer[500];
 504     for (int iter = 0; iter < 2; iter++) {
 505       char* base;
 506       int bufsize;
 507       if (iter == 0) {
 508         bufsize = sizeof(buffer);
 509         base = buffer;
 510       } else {
 511         bufsize = 30000;
 512         base = new char[bufsize];
 513       }
 514       char* p = base;
 515       char* limit = base + bufsize;
 516
 517       port::TimeVal now_tv;
 518       port::GetTimeOfDay(&now_tv, nullptr);
 519       const time_t seconds = now_tv.tv_sec;
 520       struct tm t;
 521       memset(&t, 0, sizeof(t));
 522       struct tm* ret __attribute__((__unused__));
 523       ret = port::LocalTimeR(&seconds, &t);
 524       assert(ret);
 525       p += snprintf(p, limit - p, "%04d/%02d/%02d-%02d:%02d:%02d.%06d ",
 526                     t.tm_year + 1900, t.tm_mon + 1, t.tm_mday, t.tm_hour,
 527                     t.tm_min, t.tm_sec, static_cast<int>(now_tv.tv_usec));
 528
 529       // Print the message
 530       if (p < limit) {
 531         va_list backup_ap;
 532         va_copy(backup_ap, ap);
 533         p += vsnprintf(p, limit - p, format, backup_ap);
 534         va_end(backup_ap);
 535       }
 536
 537       // Truncate to available space if necessary
 538       if (p >= limit) {
 539         if (iter == 0) {
 540           continue;  // Try again with larger buffer
 541         } else {
 542           p = limit - 1;
 543         }
 544       }
 545
 546       // Add newline if necessary
 547       if (p == base || p[-1] != '\n') {
 548         *p++ = '\n';
 549       }
 550
 551       assert(p <= limit);
 552       const size_t write_size = p - base;
 553
 554       Status s = file_->Append(Slice(base, write_size), options_, dbg_);
 555       if (s.ok()) {
 556         flush_pending_ = true;
 557         log_size_ += write_size;
 558       }
 559       uint64_t now_micros =
 560           static_cast<uint64_t>(now_tv.tv_sec) * 1000000 + now_tv.tv_usec;
 561       if (now_micros - last_flush_micros_ >= flush_every_seconds_ * 1000000) {
 562         flush_pending_ = false;
 563         last_flush_micros_ = now_micros;
 564       }
 565       if (base != buffer) {
 566         delete[] base;
 567       }
 568       break;
 569     }
 570   }
 571   size_t GetLogFileSize() const override { return log_size_; }
 572 };
 573
 574 static std::unordered_map<std::string, OptionTypeInfo> mock_fs_type_info = {
 575 #ifndef ROCKSDB_LITE
 576     {"supports_direct_io",
 577      {0, OptionType::kBoolean, OptionVerificationType::kNormal,
 578       OptionTypeFlags::kNone}},
 579 #endif  // ROCKSDB_LITE
 580 };
 581 }  // namespace
 582
 583 MockFileSystem::MockFileSystem(const std::shared_ptr<SystemClock>& clock,
 584                                bool supports_direct_io)
 585     : system_clock_(clock), supports_direct_io_(supports_direct_io) {
 586   clock_ = system_clock_.get();
 587   RegisterOptions("", &supports_direct_io_, &mock_fs_type_info);
 588 }
 589
 590 MockFileSystem::~MockFileSystem() {
 591   for (auto i = file_map_.begin(); i != file_map_.end(); ++i) {
 592     i->second->Unref();
 593   }
 594 }
 595
 596 Status MockFileSystem::PrepareOptions(const ConfigOptions& options) {
 597   Status s = FileSystem::PrepareOptions(options);
 598   if (s.ok() && system_clock_ == SystemClock::Default()) {
 599     system_clock_ = options.env->GetSystemClock();
 600     clock_ = system_clock_.get();
 601   }
 602   return s;
 603 }
 604
 605 IOStatus MockFileSystem::GetAbsolutePath(const std::string& db_path,
 606                                          const IOOptions& /*options*/,
 607                                          std::string* output_path,
 608                                          IODebugContext* /*dbg*/) {
 609   *output_path = NormalizeMockPath(db_path);
 610   if (output_path->at(0) != '/') {
 611     return IOStatus::NotSupported("GetAbsolutePath");
 612   } else {
 613     return IOStatus::OK();
 614   }
 615 }
 616
 617 std::string MockFileSystem::NormalizeMockPath(const std::string& path) {
 618   std::string p = NormalizePath(path);
 619   if (p.back() == kFilePathSeparator && p.size() > 1) {
 620     p.pop_back();
 621   }
 622   return p;
 623 }
 624
 625 // Partial implementation of the FileSystem interface.
 626 IOStatus MockFileSystem::NewSequentialFile(
 627     const std::string& fname, const FileOptions& file_opts,
 628     std::unique_ptr<FSSequentialFile>* result, IODebugContext* /*dbg*/) {
 629   auto fn = NormalizeMockPath(fname);
 630
 631   MutexLock lock(&mutex_);
 632   if (file_map_.find(fn) == file_map_.end()) {
 633     *result = nullptr;
 634     return IOStatus::PathNotFound(fn);
 635   }
 636   auto* f = file_map_[fn];
 637   if (f->is_lock_file()) {
 638     return IOStatus::InvalidArgument(fn, "Cannot open a lock file.");
 639   } else if (file_opts.use_direct_reads && !supports_direct_io_) {
 640     return IOStatus::NotSupported("Direct I/O Not Supported");
 641   } else {
 642     result->reset(new MockSequentialFile(f, file_opts));
 643     return IOStatus::OK();
 644   }
 645 }
 646
 647 IOStatus MockFileSystem::NewRandomAccessFile(
 648     const std::string& fname, const FileOptions& file_opts,
 649     std::unique_ptr<FSRandomAccessFile>* result, IODebugContext* /*dbg*/) {
 650   auto fn = NormalizeMockPath(fname);
 651   MutexLock lock(&mutex_);
 652   if (file_map_.find(fn) == file_map_.end()) {
 653     *result = nullptr;
 654     return IOStatus::PathNotFound(fn);
 655   }
 656   auto* f = file_map_[fn];
 657   if (f->is_lock_file()) {
 658     return IOStatus::InvalidArgument(fn, "Cannot open a lock file.");
 659   } else if (file_opts.use_direct_reads && !supports_direct_io_) {
 660     return IOStatus::NotSupported("Direct I/O Not Supported");
 661   } else {
 662     result->reset(new MockRandomAccessFile(f, file_opts));
 663     return IOStatus::OK();
 664   }
 665 }
 666
 667 IOStatus MockFileSystem::NewRandomRWFile(
 668     const std::string& fname, const FileOptions& /*file_opts*/,
 669     std::unique_ptr<FSRandomRWFile>* result, IODebugContext* /*dbg*/) {
 670   auto fn = NormalizeMockPath(fname);
 671   MutexLock lock(&mutex_);
 672   if (file_map_.find(fn) == file_map_.end()) {
 673     *result = nullptr;
 674     return IOStatus::PathNotFound(fn);
 675   }
 676   auto* f = file_map_[fn];
 677   if (f->is_lock_file()) {
 678     return IOStatus::InvalidArgument(fn, "Cannot open a lock file.");
 679   }
 680   result->reset(new MockRandomRWFile(f));
 681   return IOStatus::OK();
 682 }
 683
 684 IOStatus MockFileSystem::ReuseWritableFile(
 685     const std::string& fname, const std::string& old_fname,
 686     const FileOptions& options, std::unique_ptr<FSWritableFile>* result,
 687     IODebugContext* dbg) {
 688   auto s = RenameFile(old_fname, fname, IOOptions(), dbg);
 689   if (!s.ok()) {
 690     return s;
 691   } else {
 692     result->reset();
 693     return NewWritableFile(fname, options, result, dbg);
 694   }
 695 }
 696
 697 IOStatus MockFileSystem::NewWritableFile(
 698     const std::string& fname, const FileOptions& file_opts,
 699     std::unique_ptr<FSWritableFile>* result, IODebugContext* /*dbg*/) {
 700   auto fn = NormalizeMockPath(fname);
 701   MutexLock lock(&mutex_);
 702   if (file_map_.find(fn) != file_map_.end()) {
 703     DeleteFileInternal(fn);
 704   }
 705   MemFile* file = new MemFile(clock_, fn, false);
 706   file->Ref();
 707   file_map_[fn] = file;
 708   if (file_opts.use_direct_writes && !supports_direct_io_) {
 709     return IOStatus::NotSupported("Direct I/O Not Supported");
 710   } else {
 711     result->reset(new MockWritableFile(file, file_opts));
 712     return IOStatus::OK();
 713   }
 714 }
 715
 716 IOStatus MockFileSystem::ReopenWritableFile(
 717     const std::string& fname, const FileOptions& file_opts,
 718     std::unique_ptr<FSWritableFile>* result, IODebugContext* /*dbg*/) {
 719   auto fn = NormalizeMockPath(fname);
 720   MutexLock lock(&mutex_);
 721   MemFile* file = nullptr;
 722   if (file_map_.find(fn) == file_map_.end()) {
 723     file = new MemFile(clock_, fn, false);
 724     // Only take a reference when we create the file objectt
 725     file->Ref();
 726     file_map_[fn] = file;
 727   } else {
 728     file = file_map_[fn];
 729   }
 730   if (file_opts.use_direct_writes && !supports_direct_io_) {
 731     return IOStatus::NotSupported("Direct I/O Not Supported");
 732   } else {
 733     result->reset(new MockWritableFile(file, file_opts));
 734     return IOStatus::OK();
 735   }
 736 }
 737
 738 IOStatus MockFileSystem::NewDirectory(const std::string& /*name*/,
 739                                       const IOOptions& /*io_opts*/,
 740                                       std::unique_ptr<FSDirectory>* result,
 741                                       IODebugContext* /*dbg*/) {
 742   result->reset(new MockEnvDirectory());
 743   return IOStatus::OK();
 744 }
 745
 746 IOStatus MockFileSystem::FileExists(const std::string& fname,
 747                                     const IOOptions& /*io_opts*/,
 748                                     IODebugContext* /*dbg*/) {
 749   auto fn = NormalizeMockPath(fname);
 750   MutexLock lock(&mutex_);
 751   if (file_map_.find(fn) != file_map_.end()) {
 752     // File exists
 753     return IOStatus::OK();
 754   }
 755   // Now also check if fn exists as a dir
 756   for (const auto& iter : file_map_) {
 757     const std::string& filename = iter.first;
 758     if (filename.size() >= fn.size() + 1 && filename[fn.size()] == '/' &&
 759         Slice(filename).starts_with(Slice(fn))) {
 760       return IOStatus::OK();
 761     }
 762   }
 763   return IOStatus::NotFound();
 764 }
 765
 766 bool MockFileSystem::GetChildrenInternal(const std::string& dir,
 767                                          std::vector<std::string>* result) {
 768   auto d = NormalizeMockPath(dir);
 769   bool found_dir = false;
 770   result->clear();
 771   for (const auto& iter : file_map_) {
 772     const std::string& filename = iter.first;
 773
 774     if (filename == d) {
 775       found_dir = true;
 776     } else if (filename.size() >= d.size() + 1 && filename[d.size()] == '/' &&
 777                Slice(filename).starts_with(Slice(d))) {
 778       found_dir = true;
 779       size_t next_slash = filename.find('/', d.size() + 1);
 780       if (next_slash != std::string::npos) {
 781         result->push_back(
 782             filename.substr(d.size() + 1, next_slash - d.size() - 1));
 783       } else {
 784         result->push_back(filename.substr(d.size() + 1));
 785       }
 786     }
 787   }
 788   result->erase(std::unique(result->begin(), result->end()), result->end());
 789   return found_dir;
 790 }
 791
 792 IOStatus MockFileSystem::GetChildren(const std::string& dir,
 793                                      const IOOptions& /*options*/,
 794                                      std::vector<std::string>* result,
 795                                      IODebugContext* /*dbg*/) {
 796   MutexLock lock(&mutex_);
 797   bool found_dir = GetChildrenInternal(dir, result);
 798 #ifndef __clang_analyzer__
 799   return found_dir ? IOStatus::OK() : IOStatus::NotFound(dir);
 800 #else
 801   return found_dir ? IOStatus::OK() : IOStatus::NotFound();
 802 #endif
 803 }
 804
 805 void MockFileSystem::DeleteFileInternal(const std::string& fname) {
 806   assert(fname == NormalizeMockPath(fname));
 807   const auto& pair = file_map_.find(fname);
 808   if (pair != file_map_.end()) {
 809     pair->second->Unref();
 810     file_map_.erase(fname);
 811   }
 812 }
 813
 814 IOStatus MockFileSystem::DeleteFile(const std::string& fname,
 815                                     const IOOptions& /*options*/,
 816                                     IODebugContext* /*dbg*/) {
 817   auto fn = NormalizeMockPath(fname);
 818   MutexLock lock(&mutex_);
 819   if (file_map_.find(fn) == file_map_.end()) {
 820     return IOStatus::PathNotFound(fn);
 821   }
 822
 823   DeleteFileInternal(fn);
 824   return IOStatus::OK();
 825 }
 826
 827 IOStatus MockFileSystem::Truncate(const std::string& fname, size_t size,
 828                                   const IOOptions& options,
 829                                   IODebugContext* dbg) {
 830   auto fn = NormalizeMockPath(fname);
 831   MutexLock lock(&mutex_);
 832   auto iter = file_map_.find(fn);
 833   if (iter == file_map_.end()) {
 834     return IOStatus::PathNotFound(fn);
 835   }
 836   iter->second->Truncate(size, options, dbg);
 837   return IOStatus::OK();
 838 }
 839
 840 IOStatus MockFileSystem::CreateDir(const std::string& dirname,
 841                                    const IOOptions& /*options*/,
 842                                    IODebugContext* /*dbg*/) {
 843   auto dn = NormalizeMockPath(dirname);
 844   MutexLock lock(&mutex_);
 845   if (file_map_.find(dn) == file_map_.end()) {
 846     MemFile* file = new MemFile(clock_, dn, false);
 847     file->Ref();
 848     file_map_[dn] = file;
 849   } else {
 850     return IOStatus::IOError();
 851   }
 852   return IOStatus::OK();
 853 }
 854
 855 IOStatus MockFileSystem::CreateDirIfMissing(const std::string& dirname,
 856                                             const IOOptions& options,
 857                                             IODebugContext* dbg) {
 858   CreateDir(dirname, options, dbg).PermitUncheckedError();
 859   return IOStatus::OK();
 860 }
 861
 862 IOStatus MockFileSystem::DeleteDir(const std::string& dirname,
 863                                    const IOOptions& /*options*/,
 864                                    IODebugContext* /*dbg*/) {
 865   auto dir = NormalizeMockPath(dirname);
 866   MutexLock lock(&mutex_);
 867   if (file_map_.find(dir) == file_map_.end()) {
 868     return IOStatus::PathNotFound(dir);
 869   } else {
 870     std::vector<std::string> children;
 871     if (GetChildrenInternal(dir, &children)) {
 872       for (const auto& child : children) {
 873         DeleteFileInternal(child);
 874       }
 875     }
 876     DeleteFileInternal(dir);
 877     return IOStatus::OK();
 878   }
 879 }
 880
 881 IOStatus MockFileSystem::GetFileSize(const std::string& fname,
 882                                      const IOOptions& /*options*/,
 883                                      uint64_t* file_size,
 884                                      IODebugContext* /*dbg*/) {
 885   auto fn = NormalizeMockPath(fname);
 886   TEST_SYNC_POINT_CALLBACK("MockFileSystem::GetFileSize:CheckFileType", &fn);
 887   MutexLock lock(&mutex_);
 888   auto iter = file_map_.find(fn);
 889   if (iter == file_map_.end()) {
 890     return IOStatus::PathNotFound(fn);
 891   }
 892
 893   *file_size = iter->second->Size();
 894   return IOStatus::OK();
 895 }
 896
 897 IOStatus MockFileSystem::GetFileModificationTime(const std::string& fname,
 898                                                  const IOOptions& /*options*/,
 899                                                  uint64_t* time,
 900                                                  IODebugContext* /*dbg*/) {
 901   auto fn = NormalizeMockPath(fname);
 902   MutexLock lock(&mutex_);
 903   auto iter = file_map_.find(fn);
 904   if (iter == file_map_.end()) {
 905     return IOStatus::PathNotFound(fn);
 906   }
 907   *time = iter->second->ModifiedTime();
 908   return IOStatus::OK();
 909 }
 910
 911 bool MockFileSystem::RenameFileInternal(const std::string& src,
 912                                         const std::string& dest) {
 913   if (file_map_.find(src) == file_map_.end()) {
 914     return false;
 915   } else {
 916     std::vector<std::string> children;
 917     if (GetChildrenInternal(src, &children)) {
 918       for (const auto& child : children) {
 919         RenameFileInternal(src + "/" + child, dest + "/" + child);
 920       }
 921     }
 922     DeleteFileInternal(dest);
 923     file_map_[dest] = file_map_[src];
 924     file_map_.erase(src);
 925     return true;
 926   }
 927 }
 928
 929 IOStatus MockFileSystem::RenameFile(const std::string& src,
 930                                     const std::string& dest,
 931                                     const IOOptions& /*options*/,
 932                                     IODebugContext* /*dbg*/) {
 933   auto s = NormalizeMockPath(src);
 934   auto t = NormalizeMockPath(dest);
 935   MutexLock lock(&mutex_);
 936   bool found = RenameFileInternal(s, t);
 937   if (!found) {
 938     return IOStatus::PathNotFound(s);
 939   } else {
 940     return IOStatus::OK();
 941   }
 942 }
 943
 944 IOStatus MockFileSystem::LinkFile(const std::string& src,
 945                                   const std::string& dest,
 946                                   const IOOptions& /*options*/,
 947                                   IODebugContext* /*dbg*/) {
 948   auto s = NormalizeMockPath(src);
 949   auto t = NormalizeMockPath(dest);
 950   MutexLock lock(&mutex_);
 951   if (file_map_.find(s) == file_map_.end()) {
 952     return IOStatus::PathNotFound(s);
 953   }
 954
 955   DeleteFileInternal(t);
 956   file_map_[t] = file_map_[s];
 957   file_map_[t]->Ref();  // Otherwise it might get deleted when noone uses s
 958   return IOStatus::OK();
 959 }
 960
 961 IOStatus MockFileSystem::NewLogger(const std::string& fname,
 962                                    const IOOptions& io_opts,
 963                                    std::shared_ptr<Logger>* result,
 964                                    IODebugContext* dbg) {
 965   auto fn = NormalizeMockPath(fname);
 966   MutexLock lock(&mutex_);
 967   auto iter = file_map_.find(fn);
 968   MemFile* file = nullptr;
 969   if (iter == file_map_.end()) {
 970     file = new MemFile(clock_, fn, false);
 971     file->Ref();
 972     file_map_[fn] = file;
 973   } else {
 974     file = iter->second;
 975   }
 976   std::unique_ptr<FSWritableFile> f(new MockWritableFile(file, FileOptions()));
 977   result->reset(new TestMemLogger(std::move(f), clock_, io_opts, dbg));
 978   return IOStatus::OK();
 979 }
 980
 981 IOStatus MockFileSystem::LockFile(const std::string& fname,
 982                                   const IOOptions& /*options*/,
 983                                   FileLock** flock, IODebugContext* /*dbg*/) {
 984   auto fn = NormalizeMockPath(fname);
 985   {
 986     MutexLock lock(&mutex_);
 987     if (file_map_.find(fn) != file_map_.end()) {
 988       if (!file_map_[fn]->is_lock_file()) {
 989         return IOStatus::InvalidArgument(fname, "Not a lock file.");
 990       }
 991       if (!file_map_[fn]->Lock()) {
 992         return IOStatus::IOError(fn, "lock is already held.");
 993       }
 994     } else {
 995       auto* file = new MemFile(clock_, fn, true);
 996       file->Ref();
 997       file->Lock();
 998       file_map_[fn] = file;
 999     }
1000   }
1001   *flock = new MockEnvFileLock(fn);
1002   return IOStatus::OK();
1003 }
1004
1005 IOStatus MockFileSystem::UnlockFile(FileLock* flock,
1006                                     const IOOptions& /*options*/,
1007                                     IODebugContext* /*dbg*/) {
1008   std::string fn = static_cast_with_check<MockEnvFileLock>(flock)->FileName();
1009   {
1010     MutexLock lock(&mutex_);
1011     if (file_map_.find(fn) != file_map_.end()) {
1012       if (!file_map_[fn]->is_lock_file()) {
1013         return IOStatus::InvalidArgument(fn, "Not a lock file.");
1014       }
1015       file_map_[fn]->Unlock();
1016     }
1017   }
1018   delete flock;
1019   return IOStatus::OK();
1020 }
1021
1022 IOStatus MockFileSystem::GetTestDirectory(const IOOptions& /*options*/,
1023                                           std::string* path,
1024                                           IODebugContext* /*dbg*/) {
1025   *path = "/test";
1026   return IOStatus::OK();
1027 }
1028
1029 Status MockFileSystem::CorruptBuffer(const std::string& fname) {
1030   auto fn = NormalizeMockPath(fname);
1031   MutexLock lock(&mutex_);
1032   auto iter = file_map_.find(fn);
1033   if (iter == file_map_.end()) {
1034     return Status::IOError(fn, "File not found");
1035   }
1036   iter->second->CorruptBuffer();
1037   return Status::OK();
1038 }
1039
1040 MockEnv::MockEnv(Env* env, const std::shared_ptr<FileSystem>& fs,
1041                  const std::shared_ptr<SystemClock>& clock)
1042     : CompositeEnvWrapper(env, fs, clock) {}
1043
1044 MockEnv* MockEnv::Create(Env* env) {
1045   auto clock =
1046       std::make_shared<EmulatedSystemClock>(env->GetSystemClock(), true);
1047   return MockEnv::Create(env, clock);
1048 }
1049
1050 MockEnv* MockEnv::Create(Env* env, const std::shared_ptr<SystemClock>& clock) {
1051   auto fs = std::make_shared<MockFileSystem>(clock);
1052   return new MockEnv(env, fs, clock);
1053 }
1054
1055 Status MockEnv::CorruptBuffer(const std::string& fname) {
1056   auto mock = static_cast_with_check<MockFileSystem>(GetFileSystem().get());
1057   return mock->CorruptBuffer(fname);
1058 }
1059
1060 #ifndef ROCKSDB_LITE
1061 // This is to maintain the behavior before swithcing from InMemoryEnv to MockEnv
1062 Env* NewMemEnv(Env* base_env) { return MockEnv::Create(base_env); }
1063
1064 #else  // ROCKSDB_LITE
1065
1066 Env* NewMemEnv(Env* /*base_env*/) { return nullptr; }
1067
1068 #endif  // !ROCKSDB_LITE
1069
1070 }  // namespace ROCKSDB_NAMESPACE