ceph/src/rocksdb/file/file_util.cc

   1 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
   2 //  This source code is licensed under both the GPLv2 (found in the
   3 //  COPYING file in the root directory) and Apache 2.0 License
   4 //  (found in the LICENSE.Apache file in the root directory).
   5 //
   6 #include "file/file_util.h"
   7
   8 #include <string>
   9 #include <algorithm>
  10
  11 #include "file/random_access_file_reader.h"
  12 #include "file/sequence_file_reader.h"
  13 #include "file/sst_file_manager_impl.h"
  14 #include "file/writable_file_writer.h"
  15 #include "rocksdb/env.h"
  16
  17 namespace ROCKSDB_NAMESPACE {
  18
  19 // Utility function to copy a file up to a specified length
  20 IOStatus CopyFile(FileSystem* fs, const std::string& source,
  21                   const std::string& destination, uint64_t size, bool use_fsync,
  22                   const std::shared_ptr<IOTracer>& io_tracer) {
  23   const FileOptions soptions;
  24   IOStatus io_s;
  25   std::unique_ptr<SequentialFileReader> src_reader;
  26   std::unique_ptr<WritableFileWriter> dest_writer;
  27
  28   {
  29     std::unique_ptr<FSSequentialFile> srcfile;
  30     io_s = fs->NewSequentialFile(source, soptions, &srcfile, nullptr);
  31     if (!io_s.ok()) {
  32       return io_s;
  33     }
  34     std::unique_ptr<FSWritableFile> destfile;
  35     io_s = fs->NewWritableFile(destination, soptions, &destfile, nullptr);
  36     if (!io_s.ok()) {
  37       return io_s;
  38     }
  39
  40     if (size == 0) {
  41       // default argument means copy everything
  42       io_s = fs->GetFileSize(source, IOOptions(), &size, nullptr);
  43       if (!io_s.ok()) {
  44         return io_s;
  45       }
  46     }
  47     src_reader.reset(
  48         new SequentialFileReader(std::move(srcfile), source, io_tracer));
  49     dest_writer.reset(
  50         new WritableFileWriter(std::move(destfile), destination, soptions));
  51   }
  52
  53   char buffer[4096];
  54   Slice slice;
  55   while (size > 0) {
  56     size_t bytes_to_read = std::min(sizeof(buffer), static_cast<size_t>(size));
  57     io_s = status_to_io_status(src_reader->Read(bytes_to_read, &slice, buffer));
  58     if (!io_s.ok()) {
  59       return io_s;
  60     }
  61     if (slice.size() == 0) {
  62       return IOStatus::Corruption("file too small");
  63     }
  64     io_s = dest_writer->Append(slice);
  65     if (!io_s.ok()) {
  66       return io_s;
  67     }
  68     size -= slice.size();
  69   }
  70   return dest_writer->Sync(use_fsync);
  71 }
  72
  73 // Utility function to create a file with the provided contents
  74 IOStatus CreateFile(FileSystem* fs, const std::string& destination,
  75                     const std::string& contents, bool use_fsync) {
  76   const EnvOptions soptions;
  77   IOStatus io_s;
  78   std::unique_ptr<WritableFileWriter> dest_writer;
  79
  80   std::unique_ptr<FSWritableFile> destfile;
  81   io_s = fs->NewWritableFile(destination, soptions, &destfile, nullptr);
  82   if (!io_s.ok()) {
  83     return io_s;
  84   }
  85   dest_writer.reset(
  86       new WritableFileWriter(std::move(destfile), destination, soptions));
  87   io_s = dest_writer->Append(Slice(contents));
  88   if (!io_s.ok()) {
  89     return io_s;
  90   }
  91   return dest_writer->Sync(use_fsync);
  92 }
  93
  94 Status DeleteDBFile(const ImmutableDBOptions* db_options,
  95                     const std::string& fname, const std::string& dir_to_sync,
  96                     const bool force_bg, const bool force_fg) {
  97 #ifndef ROCKSDB_LITE
  98   SstFileManagerImpl* sfm =
  99       static_cast<SstFileManagerImpl*>(db_options->sst_file_manager.get());
 100   if (sfm && !force_fg) {
 101     return sfm->ScheduleFileDeletion(fname, dir_to_sync, force_bg);
 102   } else {
 103     return db_options->env->DeleteFile(fname);
 104   }
 105 #else
 106   (void)dir_to_sync;
 107   (void)force_bg;
 108   (void)force_fg;
 109   // SstFileManager is not supported in ROCKSDB_LITE
 110   // Delete file immediately
 111   return db_options->env->DeleteFile(fname);
 112 #endif
 113 }
 114
 115 bool IsWalDirSameAsDBPath(const ImmutableDBOptions* db_options) {
 116   bool same = false;
 117   assert(!db_options->db_paths.empty());
 118   Status s = db_options->env->AreFilesSame(db_options->wal_dir,
 119                                            db_options->db_paths[0].path, &same);
 120   if (s.IsNotSupported()) {
 121     same = db_options->wal_dir == db_options->db_paths[0].path;
 122   }
 123   return same;
 124 }
 125
 126 // requested_checksum_func_name brings the function name of the checksum
 127 // generator in checksum_factory. Checksum factories may use or ignore
 128 // requested_checksum_func_name.
 129 IOStatus GenerateOneFileChecksum(
 130     FileSystem* fs, const std::string& file_path,
 131     FileChecksumGenFactory* checksum_factory,
 132     const std::string& requested_checksum_func_name, std::string* file_checksum,
 133     std::string* file_checksum_func_name,
 134     size_t verify_checksums_readahead_size, bool allow_mmap_reads,
 135     std::shared_ptr<IOTracer>& io_tracer) {
 136   if (checksum_factory == nullptr) {
 137     return IOStatus::InvalidArgument("Checksum factory is invalid");
 138   }
 139   assert(file_checksum != nullptr);
 140   assert(file_checksum_func_name != nullptr);
 141
 142   FileChecksumGenContext gen_context;
 143   gen_context.requested_checksum_func_name = requested_checksum_func_name;
 144   gen_context.file_name = file_path;
 145   std::unique_ptr<FileChecksumGenerator> checksum_generator =
 146       checksum_factory->CreateFileChecksumGenerator(gen_context);
 147   if (checksum_generator == nullptr) {
 148     std::string msg =
 149         "Cannot get the file checksum generator based on the requested "
 150         "checksum function name: " +
 151         requested_checksum_func_name +
 152         " from checksum factory: " + checksum_factory->Name();
 153     return IOStatus::InvalidArgument(msg);
 154   }
 155
 156   // For backward compatable, requested_checksum_func_name can be empty.
 157   // If we give the requested checksum function name, we expect it is the
 158   // same name of the checksum generator.
 159   assert(!checksum_generator || requested_checksum_func_name.empty() ||
 160          requested_checksum_func_name == checksum_generator->Name());
 161
 162   uint64_t size;
 163   IOStatus io_s;
 164   std::unique_ptr<RandomAccessFileReader> reader;
 165   {
 166     std::unique_ptr<FSRandomAccessFile> r_file;
 167     io_s = fs->NewRandomAccessFile(file_path, FileOptions(), &r_file, nullptr);
 168     if (!io_s.ok()) {
 169       return io_s;
 170     }
 171     io_s = fs->GetFileSize(file_path, IOOptions(), &size, nullptr);
 172     if (!io_s.ok()) {
 173       return io_s;
 174     }
 175     reader.reset(new RandomAccessFileReader(std::move(r_file), file_path,
 176                                             nullptr /*Env*/, io_tracer));
 177   }
 178
 179   // Found that 256 KB readahead size provides the best performance, based on
 180   // experiments, for auto readahead. Experiment data is in PR #3282.
 181   size_t default_max_read_ahead_size = 256 * 1024;
 182   size_t readahead_size = (verify_checksums_readahead_size != 0)
 183                               ? verify_checksums_readahead_size
 184                               : default_max_read_ahead_size;
 185
 186   FilePrefetchBuffer prefetch_buffer(
 187       reader.get(), readahead_size /* readadhead_size */,
 188       readahead_size /* max_readahead_size */, !allow_mmap_reads /* enable */);
 189
 190   Slice slice;
 191   uint64_t offset = 0;
 192   IOOptions opts;
 193   while (size > 0) {
 194     size_t bytes_to_read =
 195         static_cast<size_t>(std::min(uint64_t{readahead_size}, size));
 196     if (!prefetch_buffer.TryReadFromCache(opts, offset, bytes_to_read, &slice,
 197                                           false)) {
 198       return IOStatus::Corruption("file read failed");
 199     }
 200     if (slice.size() == 0) {
 201       return IOStatus::Corruption("file too small");
 202     }
 203     checksum_generator->Update(slice.data(), slice.size());
 204     size -= slice.size();
 205     offset += slice.size();
 206   }
 207   checksum_generator->Finalize();
 208   *file_checksum = checksum_generator->GetChecksum();
 209   *file_checksum_func_name = checksum_generator->Name();
 210   return IOStatus::OK();
 211 }
 212
 213 Status DestroyDir(Env* env, const std::string& dir) {
 214   Status s;
 215   if (env->FileExists(dir).IsNotFound()) {
 216     return s;
 217   }
 218   std::vector<std::string> files_in_dir;
 219   s = env->GetChildren(dir, &files_in_dir);
 220   if (s.ok()) {
 221     for (auto& file_in_dir : files_in_dir) {
 222       if (file_in_dir == "." || file_in_dir == "..") {
 223         continue;
 224       }
 225       std::string path = dir + "/" + file_in_dir;
 226       bool is_dir = false;
 227       s = env->IsDirectory(path, &is_dir);
 228       if (s.ok()) {
 229         if (is_dir) {
 230           s = DestroyDir(env, path);
 231         } else {
 232           s = env->DeleteFile(path);
 233         }
 234       } else if (s.IsNotSupported()) {
 235         s = Status::OK();
 236       }
 237       if (!s.ok()) {
 238         // IsDirectory, etc. might not report NotFound
 239         if (s.IsNotFound() || env->FileExists(path).IsNotFound()) {
 240           // Allow files to be deleted externally
 241           s = Status::OK();
 242         } else {
 243           break;
 244         }
 245       }
 246     }
 247   }
 248
 249   if (s.ok()) {
 250     s = env->DeleteDir(dir);
 251     // DeleteDir might or might not report NotFound
 252     if (!s.ok() && (s.IsNotFound() || env->FileExists(dir).IsNotFound())) {
 253       // Allow to be deleted externally
 254       s = Status::OK();
 255     }
 256   }
 257   return s;
 258 }
 259
 260 }  // namespace ROCKSDB_NAMESPACE