ceph/src/rocksdb/env/env_basic_test.cc

   1 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file. See the AUTHORS file for names of contributors.
   4 //
   5 // Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
   6
   7 #include <algorithm>
   8 #include <memory>
   9 #include <string>
  10 #include <vector>
  11
  12 #include "env/mock_env.h"
  13 #include "rocksdb/convenience.h"
  14 #include "rocksdb/env.h"
  15 #include "rocksdb/env_encryption.h"
  16 #include "test_util/testharness.h"
  17
  18 namespace ROCKSDB_NAMESPACE {
  19
  20 // Normalizes trivial differences across Envs such that these test cases can
  21 // run on all Envs.
  22 class NormalizingEnvWrapper : public EnvWrapper {
  23  private:
  24   std::unique_ptr<Env> base_;
  25
  26  public:
  27   explicit NormalizingEnvWrapper(std::unique_ptr<Env>&& base)
  28       : EnvWrapper(base.get()), base_(std::move(base)) {}
  29   explicit NormalizingEnvWrapper(Env* base) : EnvWrapper(base) {}
  30
  31   // Removes . and .. from directory listing
  32   Status GetChildren(const std::string& dir,
  33                      std::vector<std::string>* result) override {
  34     Status status = EnvWrapper::GetChildren(dir, result);
  35     if (status.ok()) {
  36       result->erase(std::remove_if(result->begin(), result->end(),
  37                                    [](const std::string& s) {
  38                                      return s == "." || s == "..";
  39                                    }),
  40                     result->end());
  41     }
  42     return status;
  43   }
  44
  45   // Removes . and .. from directory listing
  46   Status GetChildrenFileAttributes(
  47       const std::string& dir, std::vector<FileAttributes>* result) override {
  48     Status status = EnvWrapper::GetChildrenFileAttributes(dir, result);
  49     if (status.ok()) {
  50       result->erase(std::remove_if(result->begin(), result->end(),
  51                                    [](const FileAttributes& fa) {
  52                                      return fa.name == "." || fa.name == "..";
  53                                    }),
  54                     result->end());
  55     }
  56     return status;
  57   }
  58 };
  59
  60 class EnvBasicTestWithParam : public testing::Test,
  61                               public ::testing::WithParamInterface<Env*> {
  62  public:
  63   Env* env_;
  64   const EnvOptions soptions_;
  65   std::string test_dir_;
  66
  67   EnvBasicTestWithParam() : env_(GetParam()) {
  68     test_dir_ = test::PerThreadDBPath(env_, "env_basic_test");
  69   }
  70
  71   void SetUp() override {
  72     env_->CreateDirIfMissing(test_dir_).PermitUncheckedError();
  73   }
  74
  75   void TearDown() override {
  76     std::vector<std::string> files;
  77     env_->GetChildren(test_dir_, &files).PermitUncheckedError();
  78     for (const auto& file : files) {
  79       // don't know whether it's file or directory, try both. The tests must
  80       // only create files or empty directories, so one must succeed, else the
  81       // directory's corrupted.
  82       Status s = env_->DeleteFile(test_dir_ + "/" + file);
  83       if (!s.ok()) {
  84         ASSERT_OK(env_->DeleteDir(test_dir_ + "/" + file));
  85       }
  86     }
  87   }
  88 };
  89
  90 class EnvMoreTestWithParam : public EnvBasicTestWithParam {};
  91
  92 static std::unique_ptr<Env> def_env(new NormalizingEnvWrapper(Env::Default()));
  93 INSTANTIATE_TEST_CASE_P(EnvDefault, EnvBasicTestWithParam,
  94                         ::testing::Values(def_env.get()));
  95 INSTANTIATE_TEST_CASE_P(EnvDefault, EnvMoreTestWithParam,
  96                         ::testing::Values(def_env.get()));
  97
  98 static std::unique_ptr<Env> mock_env(new MockEnv(Env::Default()));
  99 INSTANTIATE_TEST_CASE_P(MockEnv, EnvBasicTestWithParam,
 100                         ::testing::Values(mock_env.get()));
 101
 102 #ifndef ROCKSDB_LITE
 103 static Env* NewTestEncryptedEnv(Env* base, const std::string& provider_id) {
 104   std::shared_ptr<EncryptionProvider> provider;
 105   EXPECT_OK(EncryptionProvider::CreateFromString(ConfigOptions(), provider_id,
 106                                                  &provider));
 107   std::unique_ptr<Env> encrypted(NewEncryptedEnv(base, provider));
 108   return new NormalizingEnvWrapper(std::move(encrypted));
 109 }
 110
 111 // next statements run env test against default encryption code.
 112 static std::unique_ptr<Env> ctr_encrypt_env(NewTestEncryptedEnv(Env::Default(),
 113                                                                 "test://CTR"));
 114 INSTANTIATE_TEST_CASE_P(EncryptedEnv, EnvBasicTestWithParam,
 115                         ::testing::Values(ctr_encrypt_env.get()));
 116 INSTANTIATE_TEST_CASE_P(EncryptedEnv, EnvMoreTestWithParam,
 117                         ::testing::Values(ctr_encrypt_env.get()));
 118 #endif  // ROCKSDB_LITE
 119
 120 #ifndef ROCKSDB_LITE
 121 static std::unique_ptr<Env> mem_env(NewMemEnv(Env::Default()));
 122 INSTANTIATE_TEST_CASE_P(MemEnv, EnvBasicTestWithParam,
 123                         ::testing::Values(mem_env.get()));
 124
 125 namespace {
 126
 127 // Returns a vector of 0 or 1 Env*, depending whether an Env is registered for
 128 // TEST_ENV_URI.
 129 //
 130 // The purpose of returning an empty vector (instead of nullptr) is that gtest
 131 // ValuesIn() will skip running tests when given an empty collection.
 132 std::vector<Env*> GetCustomEnvs() {
 133   static Env* custom_env;
 134   static bool init = false;
 135   if (!init) {
 136     init = true;
 137     const char* uri = getenv("TEST_ENV_URI");
 138     if (uri != nullptr) {
 139       Env::LoadEnv(uri, &custom_env);
 140     }
 141   }
 142
 143   std::vector<Env*> res;
 144   if (custom_env != nullptr) {
 145     res.emplace_back(custom_env);
 146   }
 147   return res;
 148 }
 149
 150 }  // anonymous namespace
 151
 152 INSTANTIATE_TEST_CASE_P(CustomEnv, EnvBasicTestWithParam,
 153                         ::testing::ValuesIn(GetCustomEnvs()));
 154
 155 INSTANTIATE_TEST_CASE_P(CustomEnv, EnvMoreTestWithParam,
 156                         ::testing::ValuesIn(GetCustomEnvs()));
 157
 158 #endif  // ROCKSDB_LITE
 159
 160 TEST_P(EnvBasicTestWithParam, Basics) {
 161   uint64_t file_size;
 162   std::unique_ptr<WritableFile> writable_file;
 163   std::vector<std::string> children;
 164
 165   // Check that the directory is empty.
 166   ASSERT_EQ(Status::NotFound(), env_->FileExists(test_dir_ + "/non_existent"));
 167   ASSERT_TRUE(!env_->GetFileSize(test_dir_ + "/non_existent", &file_size).ok());
 168   ASSERT_OK(env_->GetChildren(test_dir_, &children));
 169   ASSERT_EQ(0U, children.size());
 170
 171   // Create a file.
 172   ASSERT_OK(env_->NewWritableFile(test_dir_ + "/f", &writable_file, soptions_));
 173   ASSERT_OK(writable_file->Close());
 174   writable_file.reset();
 175
 176   // Check that the file exists.
 177   ASSERT_OK(env_->FileExists(test_dir_ + "/f"));
 178   ASSERT_OK(env_->GetFileSize(test_dir_ + "/f", &file_size));
 179   ASSERT_EQ(0U, file_size);
 180   ASSERT_OK(env_->GetChildren(test_dir_, &children));
 181   ASSERT_EQ(1U, children.size());
 182   ASSERT_EQ("f", children[0]);
 183   ASSERT_OK(env_->DeleteFile(test_dir_ + "/f"));
 184
 185   // Write to the file.
 186   ASSERT_OK(
 187       env_->NewWritableFile(test_dir_ + "/f1", &writable_file, soptions_));
 188   ASSERT_OK(writable_file->Append("abc"));
 189   ASSERT_OK(writable_file->Close());
 190   writable_file.reset();
 191   ASSERT_OK(
 192       env_->NewWritableFile(test_dir_ + "/f2", &writable_file, soptions_));
 193   ASSERT_OK(writable_file->Close());
 194   writable_file.reset();
 195
 196   // Check for expected size.
 197   ASSERT_OK(env_->GetFileSize(test_dir_ + "/f1", &file_size));
 198   ASSERT_EQ(3U, file_size);
 199
 200   // Check that renaming works.
 201   ASSERT_TRUE(
 202       !env_->RenameFile(test_dir_ + "/non_existent", test_dir_ + "/g").ok());
 203   ASSERT_OK(env_->RenameFile(test_dir_ + "/f1", test_dir_ + "/g"));
 204   ASSERT_EQ(Status::NotFound(), env_->FileExists(test_dir_ + "/f1"));
 205   ASSERT_OK(env_->FileExists(test_dir_ + "/g"));
 206   ASSERT_OK(env_->GetFileSize(test_dir_ + "/g", &file_size));
 207   ASSERT_EQ(3U, file_size);
 208
 209   // Check that renaming overwriting works
 210   ASSERT_OK(env_->RenameFile(test_dir_ + "/f2", test_dir_ + "/g"));
 211   ASSERT_OK(env_->GetFileSize(test_dir_ + "/g", &file_size));
 212   ASSERT_EQ(0U, file_size);
 213
 214   // Check that opening non-existent file fails.
 215   std::unique_ptr<SequentialFile> seq_file;
 216   std::unique_ptr<RandomAccessFile> rand_file;
 217   ASSERT_TRUE(!env_->NewSequentialFile(test_dir_ + "/non_existent", &seq_file,
 218                                        soptions_)
 219                    .ok());
 220   ASSERT_TRUE(!seq_file);
 221   ASSERT_NOK(env_->NewRandomAccessFile(test_dir_ + "/non_existent", &rand_file,
 222                                        soptions_));
 223   ASSERT_TRUE(!rand_file);
 224
 225   // Check that deleting works.
 226   ASSERT_NOK(env_->DeleteFile(test_dir_ + "/non_existent"));
 227   ASSERT_OK(env_->DeleteFile(test_dir_ + "/g"));
 228   ASSERT_EQ(Status::NotFound(), env_->FileExists(test_dir_ + "/g"));
 229   ASSERT_OK(env_->GetChildren(test_dir_, &children));
 230   ASSERT_EQ(0U, children.size());
 231   Status s = env_->GetChildren(test_dir_ + "/non_existent", &children);
 232   ASSERT_TRUE(s.IsNotFound());
 233 }
 234
 235 TEST_P(EnvBasicTestWithParam, ReadWrite) {
 236   std::unique_ptr<WritableFile> writable_file;
 237   std::unique_ptr<SequentialFile> seq_file;
 238   std::unique_ptr<RandomAccessFile> rand_file;
 239   Slice result;
 240   char scratch[100];
 241
 242   ASSERT_OK(env_->NewWritableFile(test_dir_ + "/f", &writable_file, soptions_));
 243   ASSERT_OK(writable_file->Append("hello "));
 244   ASSERT_OK(writable_file->Append("world"));
 245   ASSERT_OK(writable_file->Close());
 246   writable_file.reset();
 247
 248   // Read sequentially.
 249   ASSERT_OK(env_->NewSequentialFile(test_dir_ + "/f", &seq_file, soptions_));
 250   ASSERT_OK(seq_file->Read(5, &result, scratch));  // Read "hello".
 251   ASSERT_EQ(0, result.compare("hello"));
 252   ASSERT_OK(seq_file->Skip(1));
 253   ASSERT_OK(seq_file->Read(1000, &result, scratch));  // Read "world".
 254   ASSERT_EQ(0, result.compare("world"));
 255   ASSERT_OK(seq_file->Read(1000, &result, scratch));  // Try reading past EOF.
 256   ASSERT_EQ(0U, result.size());
 257   ASSERT_OK(seq_file->Skip(100));  // Try to skip past end of file.
 258   ASSERT_OK(seq_file->Read(1000, &result, scratch));
 259   ASSERT_EQ(0U, result.size());
 260
 261   // Random reads.
 262   ASSERT_OK(env_->NewRandomAccessFile(test_dir_ + "/f", &rand_file, soptions_));
 263   ASSERT_OK(rand_file->Read(6, 5, &result, scratch));  // Read "world".
 264   ASSERT_EQ(0, result.compare("world"));
 265   ASSERT_OK(rand_file->Read(0, 5, &result, scratch));  // Read "hello".
 266   ASSERT_EQ(0, result.compare("hello"));
 267   ASSERT_OK(rand_file->Read(10, 100, &result, scratch));  // Read "d".
 268   ASSERT_EQ(0, result.compare("d"));
 269
 270   // Too high offset.
 271   ASSERT_TRUE(rand_file->Read(1000, 5, &result, scratch).ok());
 272 }
 273
 274 TEST_P(EnvBasicTestWithParam, Misc) {
 275   std::unique_ptr<WritableFile> writable_file;
 276   ASSERT_OK(env_->NewWritableFile(test_dir_ + "/b", &writable_file, soptions_));
 277
 278   // These are no-ops, but we test they return success.
 279   ASSERT_OK(writable_file->Sync());
 280   ASSERT_OK(writable_file->Flush());
 281   ASSERT_OK(writable_file->Close());
 282   writable_file.reset();
 283 }
 284
 285 TEST_P(EnvBasicTestWithParam, LargeWrite) {
 286   const size_t kWriteSize = 300 * 1024;
 287   char* scratch = new char[kWriteSize * 2];
 288
 289   std::string write_data;
 290   for (size_t i = 0; i < kWriteSize; ++i) {
 291     write_data.append(1, static_cast<char>(i));
 292   }
 293
 294   std::unique_ptr<WritableFile> writable_file;
 295   ASSERT_OK(env_->NewWritableFile(test_dir_ + "/f", &writable_file, soptions_));
 296   ASSERT_OK(writable_file->Append("foo"));
 297   ASSERT_OK(writable_file->Append(write_data));
 298   ASSERT_OK(writable_file->Close());
 299   writable_file.reset();
 300
 301   std::unique_ptr<SequentialFile> seq_file;
 302   Slice result;
 303   ASSERT_OK(env_->NewSequentialFile(test_dir_ + "/f", &seq_file, soptions_));
 304   ASSERT_OK(seq_file->Read(3, &result, scratch));  // Read "foo".
 305   ASSERT_EQ(0, result.compare("foo"));
 306
 307   size_t read = 0;
 308   std::string read_data;
 309   while (read < kWriteSize) {
 310     ASSERT_OK(seq_file->Read(kWriteSize - read, &result, scratch));
 311     read_data.append(result.data(), result.size());
 312     read += result.size();
 313   }
 314   ASSERT_TRUE(write_data == read_data);
 315   delete [] scratch;
 316 }
 317
 318 TEST_P(EnvMoreTestWithParam, GetModTime) {
 319   ASSERT_OK(env_->CreateDirIfMissing(test_dir_ + "/dir1"));
 320   uint64_t mtime1 = 0x0;
 321   ASSERT_OK(env_->GetFileModificationTime(test_dir_ + "/dir1", &mtime1));
 322 }
 323
 324 TEST_P(EnvMoreTestWithParam, MakeDir) {
 325   ASSERT_OK(env_->CreateDir(test_dir_ + "/j"));
 326   ASSERT_OK(env_->FileExists(test_dir_ + "/j"));
 327   std::vector<std::string> children;
 328   env_->GetChildren(test_dir_, &children);
 329   ASSERT_EQ(1U, children.size());
 330   // fail because file already exists
 331   ASSERT_TRUE(!env_->CreateDir(test_dir_ + "/j").ok());
 332   ASSERT_OK(env_->CreateDirIfMissing(test_dir_ + "/j"));
 333   ASSERT_OK(env_->DeleteDir(test_dir_ + "/j"));
 334   ASSERT_EQ(Status::NotFound(), env_->FileExists(test_dir_ + "/j"));
 335 }
 336
 337 TEST_P(EnvMoreTestWithParam, GetChildren) {
 338   // empty folder returns empty vector
 339   std::vector<std::string> children;
 340   std::vector<Env::FileAttributes> childAttr;
 341   ASSERT_OK(env_->CreateDirIfMissing(test_dir_));
 342   ASSERT_OK(env_->GetChildren(test_dir_, &children));
 343   ASSERT_OK(env_->FileExists(test_dir_));
 344   ASSERT_OK(env_->GetChildrenFileAttributes(test_dir_, &childAttr));
 345   ASSERT_EQ(0U, children.size());
 346   ASSERT_EQ(0U, childAttr.size());
 347
 348   // folder with contents returns relative path to test dir
 349   ASSERT_OK(env_->CreateDirIfMissing(test_dir_ + "/niu"));
 350   ASSERT_OK(env_->CreateDirIfMissing(test_dir_ + "/you"));
 351   ASSERT_OK(env_->CreateDirIfMissing(test_dir_ + "/guo"));
 352   ASSERT_OK(env_->GetChildren(test_dir_, &children));
 353   ASSERT_OK(env_->GetChildrenFileAttributes(test_dir_, &childAttr));
 354   ASSERT_EQ(3U, children.size());
 355   ASSERT_EQ(3U, childAttr.size());
 356   for (auto each : children) {
 357     env_->DeleteDir(test_dir_ + "/" + each).PermitUncheckedError();
 358   }  // necessary for default POSIX env
 359
 360   // non-exist directory returns IOError
 361   ASSERT_OK(env_->DeleteDir(test_dir_));
 362   ASSERT_NOK(env_->FileExists(test_dir_));
 363   ASSERT_NOK(env_->GetChildren(test_dir_, &children));
 364   ASSERT_NOK(env_->GetChildrenFileAttributes(test_dir_, &childAttr));
 365
 366   // if dir is a file, returns IOError
 367   ASSERT_OK(env_->CreateDir(test_dir_));
 368   std::unique_ptr<WritableFile> writable_file;
 369   ASSERT_OK(
 370       env_->NewWritableFile(test_dir_ + "/file", &writable_file, soptions_));
 371   ASSERT_OK(writable_file->Close());
 372   writable_file.reset();
 373   ASSERT_NOK(env_->GetChildren(test_dir_ + "/file", &children));
 374   ASSERT_EQ(0U, children.size());
 375 }
 376
 377 }  // namespace ROCKSDB_NAMESPACE
 378 int main(int argc, char** argv) {
 379   ::testing::InitGoogleTest(&argc, argv);
 380   return RUN_ALL_TESTS();
 381 }