ceph/src/rocksdb/env/file_system.cc

   1 // Copyright (c) 2019-present, Facebook, Inc.  All rights reserved.
   2 //  This source code is licensed under both the GPLv2 (found in the
   3 //  COPYING file in the root directory) and Apache 2.0 License
   4 //  (found in the LICENSE.Apache file in the root directory).
   5 //
   6 #include "rocksdb/file_system.h"
   7
   8 #include "env/composite_env_wrapper.h"
   9 #include "env/env_chroot.h"
  10 #include "env/env_encryption_ctr.h"
  11 #include "env/fs_readonly.h"
  12 #include "env/mock_env.h"
  13 #include "logging/env_logger.h"
  14 #include "options/db_options.h"
  15 #include "rocksdb/convenience.h"
  16 #include "rocksdb/utilities/customizable_util.h"
  17 #include "rocksdb/utilities/object_registry.h"
  18 #include "rocksdb/utilities/options_type.h"
  19 #include "util/string_util.h"
  20 #include "utilities/counted_fs.h"
  21 #include "utilities/env_timed.h"
  22
  23 namespace ROCKSDB_NAMESPACE {
  24
  25 FileSystem::FileSystem() {}
  26
  27 FileSystem::~FileSystem() {}
  28
  29 Status FileSystem::Load(const std::string& value,
  30                         std::shared_ptr<FileSystem>* result) {
  31   return CreateFromString(ConfigOptions(), value, result);
  32 }
  33
  34 #ifndef ROCKSDB_LITE
  35 static int RegisterBuiltinFileSystems(ObjectLibrary& library,
  36                                       const std::string& /*arg*/) {
  37   library.AddFactory<FileSystem>(
  38       TimedFileSystem::kClassName(),
  39       [](const std::string& /*uri*/, std::unique_ptr<FileSystem>* guard,
  40          std::string* /* errmsg */) {
  41         guard->reset(new TimedFileSystem(nullptr));
  42         return guard->get();
  43       });
  44   library.AddFactory<FileSystem>(
  45       ReadOnlyFileSystem::kClassName(),
  46       [](const std::string& /*uri*/, std::unique_ptr<FileSystem>* guard,
  47          std::string* /* errmsg */) {
  48         guard->reset(new ReadOnlyFileSystem(nullptr));
  49         return guard->get();
  50       });
  51   library.AddFactory<FileSystem>(
  52       EncryptedFileSystem::kClassName(),
  53       [](const std::string& /*uri*/, std::unique_ptr<FileSystem>* guard,
  54          std::string* errmsg) {
  55         Status s = NewEncryptedFileSystemImpl(nullptr, nullptr, guard);
  56         if (!s.ok()) {
  57           *errmsg = s.ToString();
  58         }
  59         return guard->get();
  60       });
  61   library.AddFactory<FileSystem>(
  62       CountedFileSystem::kClassName(),
  63       [](const std::string& /*uri*/, std::unique_ptr<FileSystem>* guard,
  64          std::string* /*errmsg*/) {
  65         guard->reset(new CountedFileSystem(FileSystem::Default()));
  66         return guard->get();
  67       });
  68   library.AddFactory<FileSystem>(
  69       MockFileSystem::kClassName(),
  70       [](const std::string& /*uri*/, std::unique_ptr<FileSystem>* guard,
  71          std::string* /*errmsg*/) {
  72         guard->reset(new MockFileSystem(SystemClock::Default()));
  73         return guard->get();
  74       });
  75 #ifndef OS_WIN
  76   library.AddFactory<FileSystem>(
  77       ChrootFileSystem::kClassName(),
  78       [](const std::string& /*uri*/, std::unique_ptr<FileSystem>* guard,
  79          std::string* /* errmsg */) {
  80         guard->reset(new ChrootFileSystem(nullptr, ""));
  81         return guard->get();
  82       });
  83 #endif  // OS_WIN
  84   size_t num_types;
  85   return static_cast<int>(library.GetFactoryCount(&num_types));
  86 }
  87 #endif  // ROCKSDB_LITE
  88
  89 Status FileSystem::CreateFromString(const ConfigOptions& config_options,
  90                                     const std::string& value,
  91                                     std::shared_ptr<FileSystem>* result) {
  92   auto default_fs = FileSystem::Default();
  93   if (default_fs->IsInstanceOf(value)) {
  94     *result = default_fs;
  95     return Status::OK();
  96   } else {
  97 #ifndef ROCKSDB_LITE
  98     static std::once_flag once;
  99     std::call_once(once, [&]() {
 100       RegisterBuiltinFileSystems(*(ObjectLibrary::Default().get()), "");
 101     });
 102 #endif  // ROCKSDB_LITE
 103     return LoadSharedObject<FileSystem>(config_options, value, nullptr, result);
 104   }
 105 }
 106
 107 IOStatus FileSystem::ReuseWritableFile(const std::string& fname,
 108                                        const std::string& old_fname,
 109                                        const FileOptions& opts,
 110                                        std::unique_ptr<FSWritableFile>* result,
 111                                        IODebugContext* dbg) {
 112   IOStatus s = RenameFile(old_fname, fname, opts.io_options, dbg);
 113   if (!s.ok()) {
 114     return s;
 115   }
 116   return NewWritableFile(fname, opts, result, dbg);
 117 }
 118
 119 IOStatus FileSystem::NewLogger(const std::string& fname,
 120                                const IOOptions& io_opts,
 121                                std::shared_ptr<Logger>* result,
 122                                IODebugContext* dbg) {
 123   FileOptions options;
 124   options.io_options = io_opts;
 125   // TODO: Tune the buffer size.
 126   options.writable_file_max_buffer_size = 1024 * 1024;
 127   std::unique_ptr<FSWritableFile> writable_file;
 128   const IOStatus status = NewWritableFile(fname, options, &writable_file, dbg);
 129   if (!status.ok()) {
 130     return status;
 131   }
 132
 133   *result = std::make_shared<EnvLogger>(std::move(writable_file), fname,
 134                                         options, Env::Default());
 135   return IOStatus::OK();
 136 }
 137
 138 FileOptions FileSystem::OptimizeForLogRead(
 139     const FileOptions& file_options) const {
 140   FileOptions optimized_file_options(file_options);
 141   optimized_file_options.use_direct_reads = false;
 142   return optimized_file_options;
 143 }
 144
 145 FileOptions FileSystem::OptimizeForManifestRead(
 146     const FileOptions& file_options) const {
 147   FileOptions optimized_file_options(file_options);
 148   optimized_file_options.use_direct_reads = false;
 149   return optimized_file_options;
 150 }
 151
 152 FileOptions FileSystem::OptimizeForLogWrite(const FileOptions& file_options,
 153                                             const DBOptions& db_options) const {
 154   FileOptions optimized_file_options(file_options);
 155   optimized_file_options.bytes_per_sync = db_options.wal_bytes_per_sync;
 156   optimized_file_options.writable_file_max_buffer_size =
 157       db_options.writable_file_max_buffer_size;
 158   return optimized_file_options;
 159 }
 160
 161 FileOptions FileSystem::OptimizeForManifestWrite(
 162     const FileOptions& file_options) const {
 163   return file_options;
 164 }
 165
 166 FileOptions FileSystem::OptimizeForCompactionTableWrite(
 167     const FileOptions& file_options,
 168     const ImmutableDBOptions& db_options) const {
 169   FileOptions optimized_file_options(file_options);
 170   optimized_file_options.use_direct_writes =
 171       db_options.use_direct_io_for_flush_and_compaction;
 172   return optimized_file_options;
 173 }
 174
 175 FileOptions FileSystem::OptimizeForCompactionTableRead(
 176     const FileOptions& file_options,
 177     const ImmutableDBOptions& db_options) const {
 178   FileOptions optimized_file_options(file_options);
 179   optimized_file_options.use_direct_reads = db_options.use_direct_reads;
 180   return optimized_file_options;
 181 }
 182
 183 FileOptions FileSystem::OptimizeForBlobFileRead(
 184     const FileOptions& file_options,
 185     const ImmutableDBOptions& db_options) const {
 186   FileOptions optimized_file_options(file_options);
 187   optimized_file_options.use_direct_reads = db_options.use_direct_reads;
 188   return optimized_file_options;
 189 }
 190
 191 IOStatus WriteStringToFile(FileSystem* fs, const Slice& data,
 192                            const std::string& fname, bool should_sync) {
 193   std::unique_ptr<FSWritableFile> file;
 194   EnvOptions soptions;
 195   IOStatus s = fs->NewWritableFile(fname, soptions, &file, nullptr);
 196   if (!s.ok()) {
 197     return s;
 198   }
 199   s = file->Append(data, IOOptions(), nullptr);
 200   if (s.ok() && should_sync) {
 201     s = file->Sync(IOOptions(), nullptr);
 202   }
 203   if (!s.ok()) {
 204     fs->DeleteFile(fname, IOOptions(), nullptr);
 205   }
 206   return s;
 207 }
 208
 209 IOStatus ReadFileToString(FileSystem* fs, const std::string& fname,
 210                           std::string* data) {
 211   FileOptions soptions;
 212   data->clear();
 213   std::unique_ptr<FSSequentialFile> file;
 214   IOStatus s = status_to_io_status(
 215       fs->NewSequentialFile(fname, soptions, &file, nullptr));
 216   if (!s.ok()) {
 217     return s;
 218   }
 219   static const int kBufferSize = 8192;
 220   char* space = new char[kBufferSize];
 221   while (true) {
 222     Slice fragment;
 223     s = file->Read(kBufferSize, IOOptions(), &fragment, space, nullptr);
 224     if (!s.ok()) {
 225       break;
 226     }
 227     data->append(fragment.data(), fragment.size());
 228     if (fragment.empty()) {
 229       break;
 230     }
 231   }
 232   delete[] space;
 233   return s;
 234 }
 235
 236 namespace {
 237 static std::unordered_map<std::string, OptionTypeInfo> fs_wrapper_type_info = {
 238 #ifndef ROCKSDB_LITE
 239     {"target",
 240      OptionTypeInfo::AsCustomSharedPtr<FileSystem>(
 241          0, OptionVerificationType::kByName, OptionTypeFlags::kDontSerialize)},
 242 #endif  // ROCKSDB_LITE
 243 };
 244 }  // namespace
 245 FileSystemWrapper::FileSystemWrapper(const std::shared_ptr<FileSystem>& t)
 246     : target_(t) {
 247   RegisterOptions("", &target_, &fs_wrapper_type_info);
 248 }
 249
 250 Status FileSystemWrapper::PrepareOptions(const ConfigOptions& options) {
 251   if (target_ == nullptr) {
 252     target_ = FileSystem::Default();
 253   }
 254   return FileSystem::PrepareOptions(options);
 255 }
 256
 257 #ifndef ROCKSDB_LITE
 258 std::string FileSystemWrapper::SerializeOptions(
 259     const ConfigOptions& config_options, const std::string& header) const {
 260   auto parent = FileSystem::SerializeOptions(config_options, "");
 261   if (config_options.IsShallow() || target_ == nullptr ||
 262       target_->IsInstanceOf(FileSystem::kDefaultName())) {
 263     return parent;
 264   } else {
 265     std::string result = header;
 266     if (!StartsWith(parent, OptionTypeInfo::kIdPropName())) {
 267       result.append(OptionTypeInfo::kIdPropName()).append("=");
 268     }
 269     result.append(parent);
 270     if (!EndsWith(result, config_options.delimiter)) {
 271       result.append(config_options.delimiter);
 272     }
 273     result.append("target=").append(target_->ToString(config_options));
 274     return result;
 275   }
 276 }
 277 #endif  // ROCKSDB_LITE
 278
 279 DirFsyncOptions::DirFsyncOptions() { reason = kDefault; }
 280
 281 DirFsyncOptions::DirFsyncOptions(std::string file_renamed_new_name) {
 282   reason = kFileRenamed;
 283   renamed_new_name = file_renamed_new_name;
 284 }
 285
 286 DirFsyncOptions::DirFsyncOptions(FsyncReason fsync_reason) {
 287   assert(fsync_reason != kFileRenamed);
 288   reason = fsync_reason;
 289 }
 290 }  // namespace ROCKSDB_NAMESPACE