ceph/src/rocksdb/options/options_parser.cc

   1 // Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
   2 //  This source code is licensed under both the GPLv2 (found in the
   3 //  COPYING file in the root directory) and Apache 2.0 License
   4 //  (found in the LICENSE.Apache file in the root directory).
   5
   6 #ifndef ROCKSDB_LITE
   7
   8 #include "options/options_parser.h"
   9
  10 #include <cmath>
  11 #include <map>
  12 #include <string>
  13 #include <utility>
  14 #include <vector>
  15
  16 #include "file/read_write_util.h"
  17 #include "file/writable_file_writer.h"
  18 #include "options/options_helper.h"
  19 #include "rocksdb/convenience.h"
  20 #include "rocksdb/db.h"
  21 #include "test_util/sync_point.h"
  22 #include "util/cast_util.h"
  23 #include "util/string_util.h"
  24
  25 #include "port/port.h"
  26
  27 namespace ROCKSDB_NAMESPACE {
  28
  29 static const std::string option_file_header =
  30     "# This is a RocksDB option file.\n"
  31     "#\n"
  32     "# For detailed file format spec, please refer to the example file\n"
  33     "# in examples/rocksdb_option_file_example.ini\n"
  34     "#\n"
  35     "\n";
  36
  37 Status PersistRocksDBOptions(const DBOptions& db_opt,
  38                              const std::vector<std::string>& cf_names,
  39                              const std::vector<ColumnFamilyOptions>& cf_opts,
  40                              const std::string& file_name, FileSystem* fs) {
  41   TEST_SYNC_POINT("PersistRocksDBOptions:start");
  42   if (cf_names.size() != cf_opts.size()) {
  43     return Status::InvalidArgument(
  44         "cf_names.size() and cf_opts.size() must be the same");
  45   }
  46   std::unique_ptr<FSWritableFile> wf;
  47
  48   Status s =
  49       fs->NewWritableFile(file_name, FileOptions(), &wf, nullptr);
  50   if (!s.ok()) {
  51     return s;
  52   }
  53   std::unique_ptr<WritableFileWriter> writable;
  54   writable.reset(new WritableFileWriter(std::move(wf), file_name, EnvOptions(),
  55                                         nullptr /* statistics */));
  56
  57   std::string options_file_content;
  58
  59   writable->Append(option_file_header + "[" +
  60                    opt_section_titles[kOptionSectionVersion] +
  61                    "]\n"
  62                    "  rocksdb_version=" +
  63                    ToString(ROCKSDB_MAJOR) + "." + ToString(ROCKSDB_MINOR) +
  64                    "." + ToString(ROCKSDB_PATCH) + "\n");
  65   writable->Append("  options_file_version=" +
  66                    ToString(ROCKSDB_OPTION_FILE_MAJOR) + "." +
  67                    ToString(ROCKSDB_OPTION_FILE_MINOR) + "\n");
  68   writable->Append("\n[" + opt_section_titles[kOptionSectionDBOptions] +
  69                    "]\n  ");
  70
  71   s = GetStringFromDBOptions(&options_file_content, db_opt, "\n  ");
  72   if (!s.ok()) {
  73     writable->Close();
  74     return s;
  75   }
  76   writable->Append(options_file_content + "\n");
  77
  78   for (size_t i = 0; i < cf_opts.size(); ++i) {
  79     // CFOptions section
  80     writable->Append("\n[" + opt_section_titles[kOptionSectionCFOptions] +
  81                      " \"" + EscapeOptionString(cf_names[i]) + "\"]\n  ");
  82     s = GetStringFromColumnFamilyOptions(&options_file_content, cf_opts[i],
  83                                          "\n  ");
  84     if (!s.ok()) {
  85       writable->Close();
  86       return s;
  87     }
  88     writable->Append(options_file_content + "\n");
  89     // TableOptions section
  90     auto* tf = cf_opts[i].table_factory.get();
  91     if (tf != nullptr) {
  92       writable->Append("[" + opt_section_titles[kOptionSectionTableOptions] +
  93                        tf->Name() + " \"" + EscapeOptionString(cf_names[i]) +
  94                        "\"]\n  ");
  95       options_file_content.clear();
  96       s = tf->GetOptionString(&options_file_content, "\n  ");
  97       if (!s.ok()) {
  98         return s;
  99       }
 100       writable->Append(options_file_content + "\n");
 101     }
 102   }
 103   writable->Sync(true /* use_fsync */);
 104   writable->Close();
 105
 106   return RocksDBOptionsParser::VerifyRocksDBOptionsFromFile(
 107       db_opt, cf_names, cf_opts, file_name, fs);
 108 }
 109
 110 RocksDBOptionsParser::RocksDBOptionsParser() { Reset(); }
 111
 112 void RocksDBOptionsParser::Reset() {
 113   db_opt_ = DBOptions();
 114   db_opt_map_.clear();
 115   cf_names_.clear();
 116   cf_opts_.clear();
 117   cf_opt_maps_.clear();
 118   has_version_section_ = false;
 119   has_db_options_ = false;
 120   has_default_cf_options_ = false;
 121   for (int i = 0; i < 3; ++i) {
 122     db_version[i] = 0;
 123     opt_file_version[i] = 0;
 124   }
 125 }
 126
 127 bool RocksDBOptionsParser::IsSection(const std::string& line) {
 128   if (line.size() < 2) {
 129     return false;
 130   }
 131   if (line[0] != '[' || line[line.size() - 1] != ']') {
 132     return false;
 133   }
 134   return true;
 135 }
 136
 137 Status RocksDBOptionsParser::ParseSection(OptionSection* section,
 138                                           std::string* title,
 139                                           std::string* argument,
 140                                           const std::string& line,
 141                                           const int line_num) {
 142   *section = kOptionSectionUnknown;
 143   // A section is of the form [<SectionName> "<SectionArg>"], where
 144   // "<SectionArg>" is optional.
 145   size_t arg_start_pos = line.find("\"");
 146   size_t arg_end_pos = line.rfind("\"");
 147   // The following if-then check tries to identify whether the input
 148   // section has the optional section argument.
 149   if (arg_start_pos != std::string::npos && arg_start_pos != arg_end_pos) {
 150     *title = TrimAndRemoveComment(line.substr(1, arg_start_pos - 1), true);
 151     *argument = UnescapeOptionString(
 152         line.substr(arg_start_pos + 1, arg_end_pos - arg_start_pos - 1));
 153   } else {
 154     *title = TrimAndRemoveComment(line.substr(1, line.size() - 2), true);
 155     *argument = "";
 156   }
 157   for (int i = 0; i < kOptionSectionUnknown; ++i) {
 158     if (title->find(opt_section_titles[i]) == 0) {
 159       if (i == kOptionSectionVersion || i == kOptionSectionDBOptions ||
 160           i == kOptionSectionCFOptions) {
 161         if (title->size() == opt_section_titles[i].size()) {
 162           // if true, then it indicats equal
 163           *section = static_cast<OptionSection>(i);
 164           return CheckSection(*section, *argument, line_num);
 165         }
 166       } else if (i == kOptionSectionTableOptions) {
 167         // This type of sections has a sufffix at the end of the
 168         // section title
 169         if (title->size() > opt_section_titles[i].size()) {
 170           *section = static_cast<OptionSection>(i);
 171           return CheckSection(*section, *argument, line_num);
 172         }
 173       }
 174     }
 175   }
 176   return Status::InvalidArgument(std::string("Unknown section ") + line);
 177 }
 178
 179 Status RocksDBOptionsParser::InvalidArgument(const int line_num,
 180                                              const std::string& message) {
 181   return Status::InvalidArgument(
 182       "[RocksDBOptionsParser Error] ",
 183       message + " (at line " + ToString(line_num) + ")");
 184 }
 185
 186 Status RocksDBOptionsParser::ParseStatement(std::string* name,
 187                                             std::string* value,
 188                                             const std::string& line,
 189                                             const int line_num) {
 190   size_t eq_pos = line.find("=");
 191   if (eq_pos == std::string::npos) {
 192     return InvalidArgument(line_num, "A valid statement must have a '='.");
 193   }
 194
 195   *name = TrimAndRemoveComment(line.substr(0, eq_pos), true);
 196   *value =
 197       TrimAndRemoveComment(line.substr(eq_pos + 1, line.size() - eq_pos - 1));
 198   if (name->empty()) {
 199     return InvalidArgument(line_num,
 200                            "A valid statement must have a variable name.");
 201   }
 202   return Status::OK();
 203 }
 204
 205 Status RocksDBOptionsParser::Parse(const std::string& file_name, FileSystem* fs,
 206                                    bool ignore_unknown_options,
 207                                    size_t file_readahead_size) {
 208   Reset();
 209
 210   std::unique_ptr<FSSequentialFile> seq_file;
 211   Status s = fs->NewSequentialFile(file_name, FileOptions(), &seq_file,
 212                                    nullptr);
 213   if (!s.ok()) {
 214     return s;
 215   }
 216
 217   SequentialFileReader sf_reader(std::move(seq_file), file_name,
 218                                  file_readahead_size);
 219
 220   OptionSection section = kOptionSectionUnknown;
 221   std::string title;
 222   std::string argument;
 223   std::unordered_map<std::string, std::string> opt_map;
 224   std::istringstream iss;
 225   std::string line;
 226   bool has_data = true;
 227   // we only support single-lined statement.
 228   for (int line_num = 1; ReadOneLine(&iss, &sf_reader, &line, &has_data, &s);
 229        ++line_num) {
 230     if (!s.ok()) {
 231       return s;
 232     }
 233     line = TrimAndRemoveComment(line);
 234     if (line.empty()) {
 235       continue;
 236     }
 237     if (IsSection(line)) {
 238       s = EndSection(section, title, argument, opt_map, ignore_unknown_options);
 239       opt_map.clear();
 240       if (!s.ok()) {
 241         return s;
 242       }
 243
 244       // If the option file is not generated by a higher minor version,
 245       // there shouldn't be any unknown option.
 246       if (ignore_unknown_options && section == kOptionSectionVersion) {
 247         if (db_version[0] < ROCKSDB_MAJOR || (db_version[0] == ROCKSDB_MAJOR &&
 248                                               db_version[1] <= ROCKSDB_MINOR)) {
 249           ignore_unknown_options = false;
 250         }
 251       }
 252
 253       s = ParseSection(&section, &title, &argument, line, line_num);
 254       if (!s.ok()) {
 255         return s;
 256       }
 257     } else {
 258       std::string name;
 259       std::string value;
 260       s = ParseStatement(&name, &value, line, line_num);
 261       if (!s.ok()) {
 262         return s;
 263       }
 264       opt_map.insert({name, value});
 265     }
 266   }
 267
 268   s = EndSection(section, title, argument, opt_map, ignore_unknown_options);
 269   opt_map.clear();
 270   if (!s.ok()) {
 271     return s;
 272   }
 273   return ValidityCheck();
 274 }
 275
 276 Status RocksDBOptionsParser::CheckSection(const OptionSection section,
 277                                           const std::string& section_arg,
 278                                           const int line_num) {
 279   if (section == kOptionSectionDBOptions) {
 280     if (has_db_options_) {
 281       return InvalidArgument(
 282           line_num,
 283           "More than one DBOption section found in the option config file");
 284     }
 285     has_db_options_ = true;
 286   } else if (section == kOptionSectionCFOptions) {
 287     bool is_default_cf = (section_arg == kDefaultColumnFamilyName);
 288     if (cf_opts_.size() == 0 && !is_default_cf) {
 289       return InvalidArgument(
 290           line_num,
 291           "Default column family must be the first CFOptions section "
 292           "in the option config file");
 293     } else if (cf_opts_.size() != 0 && is_default_cf) {
 294       return InvalidArgument(
 295           line_num,
 296           "Default column family must be the first CFOptions section "
 297           "in the optio/n config file");
 298     } else if (GetCFOptions(section_arg) != nullptr) {
 299       return InvalidArgument(
 300           line_num,
 301           "Two identical column families found in option config file");
 302     }
 303     has_default_cf_options_ |= is_default_cf;
 304   } else if (section == kOptionSectionTableOptions) {
 305     if (GetCFOptions(section_arg) == nullptr) {
 306       return InvalidArgument(
 307           line_num, std::string(
 308                         "Does not find a matched column family name in "
 309                         "TableOptions section.  Column Family Name:") +
 310                         section_arg);
 311     }
 312   } else if (section == kOptionSectionVersion) {
 313     if (has_version_section_) {
 314       return InvalidArgument(
 315           line_num,
 316           "More than one Version section found in the option config file.");
 317     }
 318     has_version_section_ = true;
 319   }
 320   return Status::OK();
 321 }
 322
 323 Status RocksDBOptionsParser::ParseVersionNumber(const std::string& ver_name,
 324                                                 const std::string& ver_string,
 325                                                 const int max_count,
 326                                                 int* version) {
 327   int version_index = 0;
 328   int current_number = 0;
 329   int current_digit_count = 0;
 330   bool has_dot = false;
 331   for (int i = 0; i < max_count; ++i) {
 332     version[i] = 0;
 333   }
 334   constexpr int kBufferSize = 200;
 335   char buffer[kBufferSize];
 336   for (size_t i = 0; i < ver_string.size(); ++i) {
 337     if (ver_string[i] == '.') {
 338       if (version_index >= max_count - 1) {
 339         snprintf(buffer, sizeof(buffer) - 1,
 340                  "A valid %s can only contains at most %d dots.",
 341                  ver_name.c_str(), max_count - 1);
 342         return Status::InvalidArgument(buffer);
 343       }
 344       if (current_digit_count == 0) {
 345         snprintf(buffer, sizeof(buffer) - 1,
 346                  "A valid %s must have at least one digit before each dot.",
 347                  ver_name.c_str());
 348         return Status::InvalidArgument(buffer);
 349       }
 350       version[version_index++] = current_number;
 351       current_number = 0;
 352       current_digit_count = 0;
 353       has_dot = true;
 354     } else if (isdigit(ver_string[i])) {
 355       current_number = current_number * 10 + (ver_string[i] - '0');
 356       current_digit_count++;
 357     } else {
 358       snprintf(buffer, sizeof(buffer) - 1,
 359                "A valid %s can only contains dots and numbers.",
 360                ver_name.c_str());
 361       return Status::InvalidArgument(buffer);
 362     }
 363   }
 364   version[version_index] = current_number;
 365   if (has_dot && current_digit_count == 0) {
 366     snprintf(buffer, sizeof(buffer) - 1,
 367              "A valid %s must have at least one digit after each dot.",
 368              ver_name.c_str());
 369     return Status::InvalidArgument(buffer);
 370   }
 371   return Status::OK();
 372 }
 373
 374 Status RocksDBOptionsParser::EndSection(
 375     const OptionSection section, const std::string& section_title,
 376     const std::string& section_arg,
 377     const std::unordered_map<std::string, std::string>& opt_map,
 378     bool ignore_unknown_options) {
 379   Status s;
 380   if (section == kOptionSectionDBOptions) {
 381     s = GetDBOptionsFromMap(DBOptions(), opt_map, &db_opt_, true,
 382                             ignore_unknown_options);
 383     if (!s.ok()) {
 384       return s;
 385     }
 386     db_opt_map_ = opt_map;
 387   } else if (section == kOptionSectionCFOptions) {
 388     // This condition should be ensured earlier in ParseSection
 389     // so we make an assertion here.
 390     assert(GetCFOptions(section_arg) == nullptr);
 391     cf_names_.emplace_back(section_arg);
 392     cf_opts_.emplace_back();
 393     s = GetColumnFamilyOptionsFromMap(ColumnFamilyOptions(), opt_map,
 394                                       &cf_opts_.back(), true,
 395                                       ignore_unknown_options);
 396     if (!s.ok()) {
 397       return s;
 398     }
 399     // keep the parsed string.
 400     cf_opt_maps_.emplace_back(opt_map);
 401   } else if (section == kOptionSectionTableOptions) {
 402     assert(GetCFOptions(section_arg) != nullptr);
 403     auto* cf_opt = GetCFOptionsImpl(section_arg);
 404     if (cf_opt == nullptr) {
 405       return Status::InvalidArgument(
 406           "The specified column family must be defined before the "
 407           "TableOptions section:",
 408           section_arg);
 409     }
 410     // Ignore error as table factory deserialization is optional
 411     s = GetTableFactoryFromMap(
 412         section_title.substr(
 413             opt_section_titles[kOptionSectionTableOptions].size()),
 414         opt_map, &(cf_opt->table_factory), ignore_unknown_options);
 415     if (!s.ok()) {
 416       return s;
 417     }
 418   } else if (section == kOptionSectionVersion) {
 419     for (const auto pair : opt_map) {
 420       if (pair.first == "rocksdb_version") {
 421         s = ParseVersionNumber(pair.first, pair.second, 3, db_version);
 422         if (!s.ok()) {
 423           return s;
 424         }
 425       } else if (pair.first == "options_file_version") {
 426         s = ParseVersionNumber(pair.first, pair.second, 2, opt_file_version);
 427         if (!s.ok()) {
 428           return s;
 429         }
 430         if (opt_file_version[0] < 1) {
 431           return Status::InvalidArgument(
 432               "A valid options_file_version must be at least 1.");
 433         }
 434       }
 435     }
 436   }
 437   return Status::OK();
 438 }
 439
 440 Status RocksDBOptionsParser::ValidityCheck() {
 441   if (!has_db_options_) {
 442     return Status::Corruption(
 443         "A RocksDB Option file must have a single DBOptions section");
 444   }
 445   if (!has_default_cf_options_) {
 446     return Status::Corruption(
 447         "A RocksDB Option file must have a single CFOptions:default section");
 448   }
 449
 450   return Status::OK();
 451 }
 452
 453 std::string RocksDBOptionsParser::TrimAndRemoveComment(const std::string& line,
 454                                                        bool trim_only) {
 455   size_t start = 0;
 456   size_t end = line.size();
 457
 458   // we only support "#" style comment
 459   if (!trim_only) {
 460     size_t search_pos = 0;
 461     while (search_pos < line.size()) {
 462       size_t comment_pos = line.find('#', search_pos);
 463       if (comment_pos == std::string::npos) {
 464         break;
 465       }
 466       if (comment_pos == 0 || line[comment_pos - 1] != '\\') {
 467         end = comment_pos;
 468         break;
 469       }
 470       search_pos = comment_pos + 1;
 471     }
 472   }
 473
 474   while (start < end && isspace(line[start]) != 0) {
 475     ++start;
 476   }
 477
 478   // start < end implies end > 0.
 479   while (start < end && isspace(line[end - 1]) != 0) {
 480     --end;
 481   }
 482
 483   if (start < end) {
 484     return line.substr(start, end - start);
 485   }
 486
 487   return "";
 488 }
 489
 490 namespace {
 491 bool AreEqualDoubles(const double a, const double b) {
 492   return (fabs(a - b) < 0.00001);
 493 }
 494 }  // namespace
 495
 496 bool AreEqualOptions(
 497     const char* opt1, const char* opt2, const OptionTypeInfo& type_info,
 498     const std::string& opt_name,
 499     const std::unordered_map<std::string, std::string>* opt_map) {
 500   const char* offset1 = opt1 + type_info.offset;
 501   const char* offset2 = opt2 + type_info.offset;
 502
 503   switch (type_info.type) {
 504     case OptionType::kBoolean:
 505       return (*reinterpret_cast<const bool*>(offset1) ==
 506               *reinterpret_cast<const bool*>(offset2));
 507     case OptionType::kInt:
 508       return (*reinterpret_cast<const int*>(offset1) ==
 509               *reinterpret_cast<const int*>(offset2));
 510     case OptionType::kInt32T:
 511       return (*reinterpret_cast<const int32_t*>(offset1) ==
 512               *reinterpret_cast<const int32_t*>(offset2));
 513     case OptionType::kInt64T:
 514       {
 515         int64_t v1, v2;
 516         GetUnaligned(reinterpret_cast<const int64_t*>(offset1), &v1);
 517         GetUnaligned(reinterpret_cast<const int64_t*>(offset2), &v2);
 518         return (v1 == v2);
 519       }
 520     case OptionType::kVectorInt:
 521       return (*reinterpret_cast<const std::vector<int>*>(offset1) ==
 522               *reinterpret_cast<const std::vector<int>*>(offset2));
 523     case OptionType::kUInt:
 524       return (*reinterpret_cast<const unsigned int*>(offset1) ==
 525               *reinterpret_cast<const unsigned int*>(offset2));
 526     case OptionType::kUInt32T:
 527       return (*reinterpret_cast<const uint32_t*>(offset1) ==
 528               *reinterpret_cast<const uint32_t*>(offset2));
 529     case OptionType::kUInt64T:
 530       {
 531         uint64_t v1, v2;
 532         GetUnaligned(reinterpret_cast<const uint64_t*>(offset1), &v1);
 533         GetUnaligned(reinterpret_cast<const uint64_t*>(offset2), &v2);
 534         return (v1 == v2);
 535       }
 536     case OptionType::kSizeT:
 537       {
 538         size_t v1, v2;
 539         GetUnaligned(reinterpret_cast<const size_t*>(offset1), &v1);
 540         GetUnaligned(reinterpret_cast<const size_t*>(offset2), &v2);
 541         return (v1 == v2);
 542       }
 543     case OptionType::kString:
 544       return (*reinterpret_cast<const std::string*>(offset1) ==
 545               *reinterpret_cast<const std::string*>(offset2));
 546     case OptionType::kDouble:
 547       return AreEqualDoubles(*reinterpret_cast<const double*>(offset1),
 548                              *reinterpret_cast<const double*>(offset2));
 549     case OptionType::kCompactionStyle:
 550       return (*reinterpret_cast<const CompactionStyle*>(offset1) ==
 551               *reinterpret_cast<const CompactionStyle*>(offset2));
 552     case OptionType::kCompactionPri:
 553       return (*reinterpret_cast<const CompactionPri*>(offset1) ==
 554               *reinterpret_cast<const CompactionPri*>(offset2));
 555     case OptionType::kCompressionType:
 556       return (*reinterpret_cast<const CompressionType*>(offset1) ==
 557               *reinterpret_cast<const CompressionType*>(offset2));
 558     case OptionType::kVectorCompressionType: {
 559       const auto* vec1 =
 560           reinterpret_cast<const std::vector<CompressionType>*>(offset1);
 561       const auto* vec2 =
 562           reinterpret_cast<const std::vector<CompressionType>*>(offset2);
 563       return (*vec1 == *vec2);
 564     }
 565     case OptionType::kChecksumType:
 566       return (*reinterpret_cast<const ChecksumType*>(offset1) ==
 567               *reinterpret_cast<const ChecksumType*>(offset2));
 568     case OptionType::kBlockBasedTableIndexType:
 569       return (
 570           *reinterpret_cast<const BlockBasedTableOptions::IndexType*>(
 571               offset1) ==
 572           *reinterpret_cast<const BlockBasedTableOptions::IndexType*>(offset2));
 573     case OptionType::kBlockBasedTableDataBlockIndexType:
 574       return (
 575           *reinterpret_cast<const BlockBasedTableOptions::DataBlockIndexType*>(
 576               offset1) ==
 577           *reinterpret_cast<const BlockBasedTableOptions::DataBlockIndexType*>(
 578               offset2));
 579     case OptionType::kBlockBasedTableIndexShorteningMode:
 580       return (
 581           *reinterpret_cast<const BlockBasedTableOptions::IndexShorteningMode*>(
 582               offset1) ==
 583           *reinterpret_cast<const BlockBasedTableOptions::IndexShorteningMode*>(
 584               offset2));
 585     case OptionType::kWALRecoveryMode:
 586       return (*reinterpret_cast<const WALRecoveryMode*>(offset1) ==
 587               *reinterpret_cast<const WALRecoveryMode*>(offset2));
 588     case OptionType::kAccessHint:
 589       return (*reinterpret_cast<const DBOptions::AccessHint*>(offset1) ==
 590               *reinterpret_cast<const DBOptions::AccessHint*>(offset2));
 591     case OptionType::kInfoLogLevel:
 592       return (*reinterpret_cast<const InfoLogLevel*>(offset1) ==
 593               *reinterpret_cast<const InfoLogLevel*>(offset2));
 594     case OptionType::kCompactionOptionsFIFO: {
 595       CompactionOptionsFIFO lhs =
 596           *reinterpret_cast<const CompactionOptionsFIFO*>(offset1);
 597       CompactionOptionsFIFO rhs =
 598           *reinterpret_cast<const CompactionOptionsFIFO*>(offset2);
 599       if (lhs.max_table_files_size == rhs.max_table_files_size &&
 600           lhs.allow_compaction == rhs.allow_compaction) {
 601         return true;
 602       }
 603       return false;
 604     }
 605     case OptionType::kCompactionOptionsUniversal: {
 606       CompactionOptionsUniversal lhs =
 607           *reinterpret_cast<const CompactionOptionsUniversal*>(offset1);
 608       CompactionOptionsUniversal rhs =
 609           *reinterpret_cast<const CompactionOptionsUniversal*>(offset2);
 610       if (lhs.size_ratio == rhs.size_ratio &&
 611           lhs.min_merge_width == rhs.min_merge_width &&
 612           lhs.max_merge_width == rhs.max_merge_width &&
 613           lhs.max_size_amplification_percent ==
 614               rhs.max_size_amplification_percent &&
 615           lhs.compression_size_percent == rhs.compression_size_percent &&
 616           lhs.stop_style == rhs.stop_style &&
 617           lhs.allow_trivial_move == rhs.allow_trivial_move) {
 618         return true;
 619       }
 620       return false;
 621     }
 622     default:
 623       if (type_info.verification == OptionVerificationType::kByName ||
 624           type_info.verification ==
 625               OptionVerificationType::kByNameAllowFromNull ||
 626           type_info.verification == OptionVerificationType::kByNameAllowNull) {
 627         std::string value1;
 628         bool result =
 629             SerializeSingleOptionHelper(offset1, type_info.type, &value1);
 630         if (result == false) {
 631           return false;
 632         }
 633         if (opt_map == nullptr) {
 634           return true;
 635         }
 636         auto iter = opt_map->find(opt_name);
 637         if (iter == opt_map->end()) {
 638           return true;
 639         } else {
 640           if (type_info.verification ==
 641               OptionVerificationType::kByNameAllowNull) {
 642             if (iter->second == kNullptrString || value1 == kNullptrString) {
 643               return true;
 644             }
 645           } else if (type_info.verification ==
 646                      OptionVerificationType::kByNameAllowFromNull) {
 647             if (iter->second == kNullptrString) {
 648               return true;
 649             }
 650           }
 651           return (value1 == iter->second);
 652         }
 653       }
 654       return false;
 655   }
 656 }
 657
 658 Status RocksDBOptionsParser::VerifyRocksDBOptionsFromFile(
 659     const DBOptions& db_opt, const std::vector<std::string>& cf_names,
 660     const std::vector<ColumnFamilyOptions>& cf_opts,
 661     const std::string& file_name, FileSystem* fs,
 662     OptionsSanityCheckLevel sanity_check_level, bool ignore_unknown_options) {
 663   // We infer option file readhead size from log readahead size.
 664   // If it is not given, use 512KB.
 665   size_t file_readahead_size = db_opt.log_readahead_size;
 666   if (file_readahead_size == 0) {
 667     const size_t kDefaultOptionFileReadAheadSize = 512 * 1024;
 668     file_readahead_size = kDefaultOptionFileReadAheadSize;
 669   }
 670
 671   RocksDBOptionsParser parser;
 672   Status s =
 673       parser.Parse(file_name, fs, ignore_unknown_options, file_readahead_size);
 674   if (!s.ok()) {
 675     return s;
 676   }
 677
 678   // Verify DBOptions
 679   s = VerifyDBOptions(db_opt, *parser.db_opt(), parser.db_opt_map(),
 680                       sanity_check_level);
 681   if (!s.ok()) {
 682     return s;
 683   }
 684
 685   // Verify ColumnFamily Name
 686   if (cf_names.size() != parser.cf_names()->size()) {
 687     if (sanity_check_level >= kSanityLevelLooselyCompatible) {
 688       return Status::InvalidArgument(
 689           "[RocksDBOptionParser Error] The persisted options does not have "
 690           "the same number of column family names as the db instance.");
 691     } else if (cf_opts.size() > parser.cf_opts()->size()) {
 692       return Status::InvalidArgument(
 693           "[RocksDBOptionsParser Error]",
 694           "The persisted options file has less number of column family "
 695           "names than that of the specified one.");
 696     }
 697   }
 698   for (size_t i = 0; i < cf_names.size(); ++i) {
 699     if (cf_names[i] != parser.cf_names()->at(i)) {
 700       return Status::InvalidArgument(
 701           "[RocksDBOptionParser Error] The persisted options and the db"
 702           "instance does not have the same name for column family ",
 703           ToString(i));
 704     }
 705   }
 706
 707   // Verify Column Family Options
 708   if (cf_opts.size() != parser.cf_opts()->size()) {
 709     if (sanity_check_level >= kSanityLevelLooselyCompatible) {
 710       return Status::InvalidArgument(
 711           "[RocksDBOptionsParser Error]",
 712           "The persisted options does not have the same number of "
 713           "column families as the db instance.");
 714     } else if (cf_opts.size() > parser.cf_opts()->size()) {
 715       return Status::InvalidArgument(
 716           "[RocksDBOptionsParser Error]",
 717           "The persisted options file has less number of column families "
 718           "than that of the specified number.");
 719     }
 720   }
 721   for (size_t i = 0; i < cf_opts.size(); ++i) {
 722     s = VerifyCFOptions(cf_opts[i], parser.cf_opts()->at(i),
 723                         &(parser.cf_opt_maps()->at(i)), sanity_check_level);
 724     if (!s.ok()) {
 725       return s;
 726     }
 727     s = VerifyTableFactory(cf_opts[i].table_factory.get(),
 728                            parser.cf_opts()->at(i).table_factory.get(),
 729                            sanity_check_level);
 730     if (!s.ok()) {
 731       return s;
 732     }
 733   }
 734
 735   return Status::OK();
 736 }
 737
 738 Status RocksDBOptionsParser::VerifyDBOptions(
 739     const DBOptions& base_opt, const DBOptions& persisted_opt,
 740     const std::unordered_map<std::string, std::string>* /*opt_map*/,
 741     OptionsSanityCheckLevel sanity_check_level) {
 742   for (auto pair : db_options_type_info) {
 743     if (pair.second.verification == OptionVerificationType::kDeprecated) {
 744       // We skip checking deprecated variables as they might
 745       // contain random values since they might not be initialized
 746       continue;
 747     }
 748     if (DBOptionSanityCheckLevel(pair.first) <= sanity_check_level) {
 749       if (!AreEqualOptions(reinterpret_cast<const char*>(&base_opt),
 750                            reinterpret_cast<const char*>(&persisted_opt),
 751                            pair.second, pair.first, nullptr)) {
 752         constexpr size_t kBufferSize = 2048;
 753         char buffer[kBufferSize];
 754         std::string base_value;
 755         std::string persisted_value;
 756         SerializeSingleOptionHelper(
 757             reinterpret_cast<const char*>(&base_opt) + pair.second.offset,
 758             pair.second.type, &base_value);
 759         SerializeSingleOptionHelper(
 760             reinterpret_cast<const char*>(&persisted_opt) + pair.second.offset,
 761             pair.second.type, &persisted_value);
 762         snprintf(buffer, sizeof(buffer),
 763                  "[RocksDBOptionsParser]: "
 764                  "failed the verification on DBOptions::%s --- "
 765                  "The specified one is %s while the persisted one is %s.\n",
 766                  pair.first.c_str(), base_value.c_str(),
 767                  persisted_value.c_str());
 768         return Status::InvalidArgument(Slice(buffer, strlen(buffer)));
 769       }
 770     }
 771   }
 772   return Status::OK();
 773 }
 774
 775 Status RocksDBOptionsParser::VerifyCFOptions(
 776     const ColumnFamilyOptions& base_opt,
 777     const ColumnFamilyOptions& persisted_opt,
 778     const std::unordered_map<std::string, std::string>* persisted_opt_map,
 779     OptionsSanityCheckLevel sanity_check_level) {
 780   for (auto& pair : cf_options_type_info) {
 781     if (pair.second.verification == OptionVerificationType::kDeprecated) {
 782       // We skip checking deprecated variables as they might
 783       // contain random values since they might not be initialized
 784       continue;
 785     }
 786     if (CFOptionSanityCheckLevel(pair.first) <= sanity_check_level) {
 787       if (!AreEqualOptions(reinterpret_cast<const char*>(&base_opt),
 788                            reinterpret_cast<const char*>(&persisted_opt),
 789                            pair.second, pair.first, persisted_opt_map)) {
 790         constexpr size_t kBufferSize = 2048;
 791         char buffer[kBufferSize];
 792         std::string base_value;
 793         std::string persisted_value;
 794         SerializeSingleOptionHelper(
 795             reinterpret_cast<const char*>(&base_opt) + pair.second.offset,
 796             pair.second.type, &base_value);
 797         SerializeSingleOptionHelper(
 798             reinterpret_cast<const char*>(&persisted_opt) + pair.second.offset,
 799             pair.second.type, &persisted_value);
 800         snprintf(buffer, sizeof(buffer),
 801                  "[RocksDBOptionsParser]: "
 802                  "failed the verification on ColumnFamilyOptions::%s --- "
 803                  "The specified one is %s while the persisted one is %s.\n",
 804                  pair.first.c_str(), base_value.c_str(),
 805                  persisted_value.c_str());
 806         return Status::InvalidArgument(Slice(buffer, sizeof(buffer)));
 807       }
 808     }
 809   }
 810   return Status::OK();
 811 }
 812
 813 Status RocksDBOptionsParser::VerifyTableFactory(
 814     const TableFactory* base_tf, const TableFactory* file_tf,
 815     OptionsSanityCheckLevel sanity_check_level) {
 816   if (base_tf && file_tf) {
 817     if (sanity_check_level > kSanityLevelNone &&
 818         std::string(base_tf->Name()) != std::string(file_tf->Name())) {
 819       return Status::Corruption(
 820           "[RocksDBOptionsParser]: "
 821           "failed the verification on TableFactory->Name()");
 822     }
 823     if (base_tf->Name() == BlockBasedTableFactory::kName) {
 824       return VerifyBlockBasedTableFactory(
 825           static_cast_with_check<const BlockBasedTableFactory,
 826                                  const TableFactory>(base_tf),
 827           static_cast_with_check<const BlockBasedTableFactory,
 828                                  const TableFactory>(file_tf),
 829           sanity_check_level);
 830     }
 831     // TODO(yhchiang): add checks for other table factory types
 832   } else {
 833     // TODO(yhchiang): further support sanity check here
 834   }
 835   return Status::OK();
 836 }
 837 }  // namespace ROCKSDB_NAMESPACE
 838
 839 #endif  // !ROCKSDB_LITE