ceph/src/rocksdb/tools/sst_dump_test.cc

   1 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
   2 //  This source code is licensed under both the GPLv2 (found in the
   3 //  COPYING file in the root directory) and Apache 2.0 License
   4 //  (found in the LICENSE.Apache file in the root directory).
   5 //
   6 // Copyright (c) 2012 The LevelDB Authors. All rights reserved.
   7 // Use of this source code is governed by a BSD-style license that can be
   8 // found in the LICENSE file. See the AUTHORS file for names of contributors.
   9 #ifndef ROCKSDB_LITE
  10
  11 #include <stdint.h>
  12
  13 #include "file/random_access_file_reader.h"
  14 #include "port/stack_trace.h"
  15 #include "rocksdb/convenience.h"
  16 #include "rocksdb/filter_policy.h"
  17 #include "rocksdb/sst_dump_tool.h"
  18 #include "table/block_based/block_based_table_factory.h"
  19 #include "table/table_builder.h"
  20 #include "test_util/testharness.h"
  21 #include "test_util/testutil.h"
  22
  23 namespace ROCKSDB_NAMESPACE {
  24
  25 const uint32_t kOptLength = 1024;
  26
  27 namespace {
  28 static std::string MakeKey(int i) {
  29   char buf[100];
  30   snprintf(buf, sizeof(buf), "k_%04d", i);
  31   InternalKey key(std::string(buf), 0, ValueType::kTypeValue);
  32   return key.Encode().ToString();
  33 }
  34
  35 static std::string MakeKeyWithTimeStamp(int i, uint64_t ts) {
  36   char buf[100];
  37   snprintf(buf, sizeof(buf), "k_%04d", i);
  38   return test::KeyStr(ts, std::string(buf), /*seq=*/0, kTypeValue);
  39 }
  40
  41 static std::string MakeValue(int i) {
  42   char buf[100];
  43   snprintf(buf, sizeof(buf), "v_%04d", i);
  44   InternalKey key(std::string(buf), 0, ValueType::kTypeValue);
  45   return key.Encode().ToString();
  46 }
  47
  48 void cleanup(const Options& opts, const std::string& file_name) {
  49   Env* env = opts.env;
  50   ASSERT_OK(env->DeleteFile(file_name));
  51   std::string outfile_name = file_name.substr(0, file_name.length() - 4);
  52   outfile_name.append("_dump.txt");
  53   env->DeleteFile(outfile_name).PermitUncheckedError();
  54 }
  55 }  // namespace
  56
  57 // Test for sst dump tool "raw" mode
  58 class SSTDumpToolTest : public testing::Test {
  59   std::string test_dir_;
  60   Env* env_;
  61   std::shared_ptr<Env> env_guard_;
  62
  63  public:
  64   SSTDumpToolTest() : env_(Env::Default()) {
  65     EXPECT_OK(test::CreateEnvFromSystem(ConfigOptions(), &env_, &env_guard_));
  66     test_dir_ = test::PerThreadDBPath(env_, "sst_dump_test_db");
  67     Status s = env_->CreateDirIfMissing(test_dir_);
  68     EXPECT_OK(s);
  69   }
  70
  71   ~SSTDumpToolTest() override {
  72     if (getenv("KEEP_DB")) {
  73       fprintf(stdout, "Data is still at %s\n", test_dir_.c_str());
  74     } else {
  75       EXPECT_OK(env_->DeleteDir(test_dir_));
  76     }
  77   }
  78
  79   Env* env() { return env_; }
  80
  81   std::string MakeFilePath(const std::string& file_name) const {
  82     std::string path(test_dir_);
  83     path.append("/").append(file_name);
  84     return path;
  85   }
  86
  87   template <std::size_t N>
  88   void PopulateCommandArgs(const std::string& file_path, const char* command,
  89                            char* (&usage)[N]) const {
  90     for (int i = 0; i < static_cast<int>(N); ++i) {
  91       usage[i] = new char[kOptLength];
  92     }
  93     snprintf(usage[0], kOptLength, "./sst_dump");
  94     snprintf(usage[1], kOptLength, "%s", command);
  95     snprintf(usage[2], kOptLength, "--file=%s", file_path.c_str());
  96   }
  97
  98   void createSST(const Options& opts, const std::string& file_name) {
  99     Env* test_env = opts.env;
 100     FileOptions file_options(opts);
 101     ReadOptions read_options;
 102     const ImmutableOptions imoptions(opts);
 103     const MutableCFOptions moptions(opts);
 104     ROCKSDB_NAMESPACE::InternalKeyComparator ikc(opts.comparator);
 105     std::unique_ptr<TableBuilder> tb;
 106
 107     IntTblPropCollectorFactories int_tbl_prop_collector_factories;
 108     std::unique_ptr<WritableFileWriter> file_writer;
 109     ASSERT_OK(WritableFileWriter::Create(test_env->GetFileSystem(), file_name,
 110                                          file_options, &file_writer, nullptr));
 111
 112     std::string column_family_name;
 113     int unknown_level = -1;
 114     tb.reset(opts.table_factory->NewTableBuilder(
 115         TableBuilderOptions(
 116             imoptions, moptions, ikc, &int_tbl_prop_collector_factories,
 117             CompressionType::kNoCompression, CompressionOptions(),
 118             TablePropertiesCollectorFactory::Context::kUnknownColumnFamily,
 119             column_family_name, unknown_level),
 120         file_writer.get()));
 121
 122     // Populate slightly more than 1K keys
 123     uint32_t num_keys = kNumKey;
 124     const char* comparator_name = ikc.user_comparator()->Name();
 125     if (strcmp(comparator_name, ReverseBytewiseComparator()->Name()) == 0) {
 126       for (int32_t i = num_keys; i >= 0; i--) {
 127         tb->Add(MakeKey(i), MakeValue(i));
 128       }
 129     } else if (strcmp(comparator_name,
 130                       test::BytewiseComparatorWithU64TsWrapper()->Name()) ==
 131                0) {
 132       for (uint32_t i = 0; i < num_keys; i++) {
 133         tb->Add(MakeKeyWithTimeStamp(i, 100 + i), MakeValue(i));
 134       }
 135     } else {
 136       for (uint32_t i = 0; i < num_keys; i++) {
 137         tb->Add(MakeKey(i), MakeValue(i));
 138       }
 139     }
 140     ASSERT_OK(tb->Finish());
 141     ASSERT_OK(file_writer->Close());
 142   }
 143
 144  protected:
 145   constexpr static int kNumKey = 1024;
 146 };
 147
 148 constexpr int SSTDumpToolTest::kNumKey;
 149
 150 TEST_F(SSTDumpToolTest, HelpAndVersion) {
 151   Options opts;
 152   opts.env = env();
 153
 154   ROCKSDB_NAMESPACE::SSTDumpTool tool;
 155
 156   static const char* help[] = {"./sst_dump", "--help"};
 157   ASSERT_TRUE(!tool.Run(2, help, opts));
 158   static const char* version[] = {"./sst_dump", "--version"};
 159   ASSERT_TRUE(!tool.Run(2, version, opts));
 160   static const char* bad[] = {"./sst_dump", "--not_an_option"};
 161   ASSERT_TRUE(tool.Run(2, bad, opts));
 162 }
 163
 164 TEST_F(SSTDumpToolTest, EmptyFilter) {
 165   Options opts;
 166   opts.env = env();
 167   std::string file_path = MakeFilePath("rocksdb_sst_test.sst");
 168   createSST(opts, file_path);
 169
 170   char* usage[3];
 171   PopulateCommandArgs(file_path, "--command=raw", usage);
 172
 173   ROCKSDB_NAMESPACE::SSTDumpTool tool;
 174   ASSERT_TRUE(!tool.Run(3, usage, opts));
 175
 176   cleanup(opts, file_path);
 177   for (int i = 0; i < 3; i++) {
 178     delete[] usage[i];
 179   }
 180 }
 181
 182 TEST_F(SSTDumpToolTest, SstDumpReverseBytewiseComparator) {
 183   Options opts;
 184   opts.env = env();
 185   opts.comparator = ReverseBytewiseComparator();
 186   BlockBasedTableOptions table_opts;
 187   table_opts.filter_policy.reset(
 188       ROCKSDB_NAMESPACE::NewBloomFilterPolicy(10, false));
 189   opts.table_factory.reset(new BlockBasedTableFactory(table_opts));
 190   std::string file_path =
 191       MakeFilePath("rocksdb_sst_reverse_bytewise_comparator.sst");
 192   createSST(opts, file_path);
 193
 194   char* usage[3];
 195   PopulateCommandArgs(file_path, "--command=raw", usage);
 196
 197   ROCKSDB_NAMESPACE::SSTDumpTool tool;
 198   ASSERT_TRUE(!tool.Run(3, usage, opts));
 199
 200   cleanup(opts, file_path);
 201   for (int i = 0; i < 3; i++) {
 202     delete[] usage[i];
 203   }
 204 }
 205
 206 TEST_F(SSTDumpToolTest, SstDumpComparatorWithU64Ts) {
 207   Options opts;
 208   opts.env = env();
 209   opts.comparator = test::BytewiseComparatorWithU64TsWrapper();
 210   BlockBasedTableOptions table_opts;
 211   table_opts.filter_policy.reset(
 212       ROCKSDB_NAMESPACE::NewBloomFilterPolicy(10, false));
 213   opts.table_factory.reset(new BlockBasedTableFactory(table_opts));
 214   std::string file_path =
 215       MakeFilePath("rocksdb_sst_comparator_with_u64_ts.sst");
 216   createSST(opts, file_path);
 217
 218   char* usage[3];
 219   PopulateCommandArgs(file_path, "--command=raw", usage);
 220
 221   ROCKSDB_NAMESPACE::SSTDumpTool tool;
 222   ASSERT_TRUE(!tool.Run(3, usage, opts));
 223
 224   cleanup(opts, file_path);
 225   for (int i = 0; i < 3; i++) {
 226     delete[] usage[i];
 227   }
 228 }
 229
 230 TEST_F(SSTDumpToolTest, FilterBlock) {
 231   Options opts;
 232   opts.env = env();
 233   BlockBasedTableOptions table_opts;
 234   table_opts.filter_policy.reset(
 235       ROCKSDB_NAMESPACE::NewBloomFilterPolicy(10, true));
 236   opts.table_factory.reset(new BlockBasedTableFactory(table_opts));
 237   std::string file_path = MakeFilePath("rocksdb_sst_test.sst");
 238   createSST(opts, file_path);
 239
 240   char* usage[3];
 241   PopulateCommandArgs(file_path, "--command=raw", usage);
 242
 243   ROCKSDB_NAMESPACE::SSTDumpTool tool;
 244   ASSERT_TRUE(!tool.Run(3, usage, opts));
 245
 246   cleanup(opts, file_path);
 247   for (int i = 0; i < 3; i++) {
 248     delete[] usage[i];
 249   }
 250 }
 251
 252 TEST_F(SSTDumpToolTest, FullFilterBlock) {
 253   Options opts;
 254   opts.env = env();
 255   BlockBasedTableOptions table_opts;
 256   table_opts.filter_policy.reset(
 257       ROCKSDB_NAMESPACE::NewBloomFilterPolicy(10, false));
 258   opts.table_factory.reset(new BlockBasedTableFactory(table_opts));
 259   std::string file_path = MakeFilePath("rocksdb_sst_test.sst");
 260   createSST(opts, file_path);
 261
 262   char* usage[3];
 263   PopulateCommandArgs(file_path, "--command=raw", usage);
 264
 265   ROCKSDB_NAMESPACE::SSTDumpTool tool;
 266   ASSERT_TRUE(!tool.Run(3, usage, opts));
 267
 268   cleanup(opts, file_path);
 269   for (int i = 0; i < 3; i++) {
 270     delete[] usage[i];
 271   }
 272 }
 273
 274 TEST_F(SSTDumpToolTest, GetProperties) {
 275   Options opts;
 276   opts.env = env();
 277   BlockBasedTableOptions table_opts;
 278   table_opts.filter_policy.reset(
 279       ROCKSDB_NAMESPACE::NewBloomFilterPolicy(10, false));
 280   opts.table_factory.reset(new BlockBasedTableFactory(table_opts));
 281   std::string file_path = MakeFilePath("rocksdb_sst_test.sst");
 282   createSST(opts, file_path);
 283
 284   char* usage[3];
 285   PopulateCommandArgs(file_path, "--show_properties", usage);
 286
 287   ROCKSDB_NAMESPACE::SSTDumpTool tool;
 288   ASSERT_TRUE(!tool.Run(3, usage, opts));
 289
 290   cleanup(opts, file_path);
 291   for (int i = 0; i < 3; i++) {
 292     delete[] usage[i];
 293   }
 294 }
 295
 296 TEST_F(SSTDumpToolTest, CompressedSizes) {
 297   Options opts;
 298   opts.env = env();
 299   BlockBasedTableOptions table_opts;
 300   table_opts.filter_policy.reset(
 301       ROCKSDB_NAMESPACE::NewBloomFilterPolicy(10, false));
 302   opts.table_factory.reset(new BlockBasedTableFactory(table_opts));
 303   std::string file_path = MakeFilePath("rocksdb_sst_test.sst");
 304   createSST(opts, file_path);
 305
 306   char* usage[3];
 307   PopulateCommandArgs(file_path, "--command=recompress", usage);
 308
 309   ROCKSDB_NAMESPACE::SSTDumpTool tool;
 310   ASSERT_TRUE(!tool.Run(3, usage, opts));
 311
 312   cleanup(opts, file_path);
 313   for (int i = 0; i < 3; i++) {
 314     delete[] usage[i];
 315   }
 316 }
 317
 318 TEST_F(SSTDumpToolTest, MemEnv) {
 319   std::unique_ptr<Env> mem_env(NewMemEnv(env()));
 320   Options opts;
 321   opts.env = mem_env.get();
 322   std::string file_path = MakeFilePath("rocksdb_sst_test.sst");
 323   createSST(opts, file_path);
 324
 325   char* usage[3];
 326   PopulateCommandArgs(file_path, "--command=verify_checksum", usage);
 327
 328   ROCKSDB_NAMESPACE::SSTDumpTool tool;
 329   ASSERT_TRUE(!tool.Run(3, usage, opts));
 330
 331   cleanup(opts, file_path);
 332   for (int i = 0; i < 3; i++) {
 333     delete[] usage[i];
 334   }
 335 }
 336
 337 TEST_F(SSTDumpToolTest, ReadaheadSize) {
 338   Options opts;
 339   opts.env = env();
 340   std::string file_path = MakeFilePath("rocksdb_sst_test.sst");
 341   createSST(opts, file_path);
 342
 343   char* usage[4];
 344   PopulateCommandArgs(file_path, "--command=verify", usage);
 345   snprintf(usage[3], kOptLength, "--readahead_size=4000000");
 346
 347   int num_reads = 0;
 348   SyncPoint::GetInstance()->SetCallBack("RandomAccessFileReader::Read",
 349                                         [&](void*) { num_reads++; });
 350   SyncPoint::GetInstance()->EnableProcessing();
 351
 352   SSTDumpTool tool;
 353   ASSERT_TRUE(!tool.Run(4, usage, opts));
 354
 355   // The file is approximately 10MB. Readahead is 4MB.
 356   // We usually need 3 reads + one metadata read.
 357   // One extra read is needed before opening the file for metadata.
 358   ASSERT_EQ(5, num_reads);
 359
 360   SyncPoint::GetInstance()->ClearAllCallBacks();
 361   SyncPoint::GetInstance()->DisableProcessing();
 362
 363   cleanup(opts, file_path);
 364   for (int i = 0; i < 4; i++) {
 365     delete[] usage[i];
 366   }
 367 }
 368
 369 TEST_F(SSTDumpToolTest, NoSstFile) {
 370   Options opts;
 371   opts.env = env();
 372   std::string file_path = MakeFilePath("no_such_file.sst");
 373   char* usage[3];
 374   PopulateCommandArgs(file_path, "", usage);
 375   ROCKSDB_NAMESPACE::SSTDumpTool tool;
 376   for (const auto& command :
 377        {"--command=check", "--command=dump", "--command=raw",
 378         "--command=verify", "--command=recompress", "--command=verify_checksum",
 379         "--show_properties"}) {
 380     snprintf(usage[1], kOptLength, "%s", command);
 381     ASSERT_TRUE(tool.Run(3, usage, opts));
 382   }
 383   for (int i = 0; i < 3; i++) {
 384     delete[] usage[i];
 385   }
 386 }
 387
 388 TEST_F(SSTDumpToolTest, ValidSSTPath) {
 389   Options opts;
 390   opts.env = env();
 391   char* usage[3];
 392   PopulateCommandArgs("", "", usage);
 393   SSTDumpTool tool;
 394   std::string file_not_exists = MakeFilePath("file_not_exists.sst");
 395   std::string sst_file = MakeFilePath("rocksdb_sst_test.sst");
 396   createSST(opts, sst_file);
 397   std::string text_file = MakeFilePath("text_file");
 398   ASSERT_OK(WriteStringToFile(opts.env, "Hello World!", text_file));
 399   std::string fake_sst = MakeFilePath("fake_sst.sst");
 400   ASSERT_OK(WriteStringToFile(opts.env, "Not an SST file!", fake_sst));
 401
 402   for (const auto& command_arg : {"--command=verify", "--command=identify"}) {
 403     snprintf(usage[1], kOptLength, "%s", command_arg);
 404
 405     snprintf(usage[2], kOptLength, "--file=%s", file_not_exists.c_str());
 406     ASSERT_TRUE(tool.Run(3, usage, opts));
 407
 408     snprintf(usage[2], kOptLength, "--file=%s", sst_file.c_str());
 409     ASSERT_TRUE(!tool.Run(3, usage, opts));
 410
 411     snprintf(usage[2], kOptLength, "--file=%s", text_file.c_str());
 412     ASSERT_TRUE(tool.Run(3, usage, opts));
 413
 414     snprintf(usage[2], kOptLength, "--file=%s", fake_sst.c_str());
 415     ASSERT_TRUE(tool.Run(3, usage, opts));
 416   }
 417   ASSERT_OK(opts.env->DeleteFile(sst_file));
 418   ASSERT_OK(opts.env->DeleteFile(text_file));
 419   ASSERT_OK(opts.env->DeleteFile(fake_sst));
 420
 421   for (int i = 0; i < 3; i++) {
 422     delete[] usage[i];
 423   }
 424 }
 425
 426 TEST_F(SSTDumpToolTest, RawOutput) {
 427   Options opts;
 428   opts.env = env();
 429   std::string file_path = MakeFilePath("rocksdb_sst_test.sst");
 430   createSST(opts, file_path);
 431
 432   char* usage[3];
 433   PopulateCommandArgs(file_path, "--command=raw", usage);
 434
 435   ROCKSDB_NAMESPACE::SSTDumpTool tool;
 436   ASSERT_TRUE(!tool.Run(3, usage, opts));
 437
 438   const std::string raw_path = MakeFilePath("rocksdb_sst_test_dump.txt");
 439   std::ifstream raw_file(raw_path);
 440
 441   std::string tp;
 442   bool is_data_block = false;
 443   int key_count = 0;
 444   while (getline(raw_file, tp)) {
 445     if (tp.find("Data Block #") != std::string::npos) {
 446       is_data_block = true;
 447     }
 448
 449     if (is_data_block && tp.find("HEX") != std::string::npos) {
 450       key_count++;
 451     }
 452   }
 453
 454   ASSERT_EQ(kNumKey, key_count);
 455
 456   raw_file.close();
 457
 458   cleanup(opts, file_path);
 459   for (int i = 0; i < 3; i++) {
 460     delete[] usage[i];
 461   }
 462 }
 463
 464 }  // namespace ROCKSDB_NAMESPACE
 465
 466 int main(int argc, char** argv) {
 467   ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
 468   ::testing::InitGoogleTest(&argc, argv);
 469   RegisterCustomObjects(argc, argv);
 470   return RUN_ALL_TESTS();
 471 }
 472
 473 #else
 474 #include <stdio.h>
 475
 476 int main(int /*argc*/, char** /*argv*/) {
 477   fprintf(stderr, "SKIPPED as SSTDumpTool is not supported in ROCKSDB_LITE\n");
 478   return 0;
 479 }
 480
 481 #endif  // !ROCKSDB_LITE  return RUN_ALL_TESTS();