ceph/src/rocksdb/test_util/testutil.cc

   1 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
   2 //  This source code is licensed under both the GPLv2 (found in the
   3 //  COPYING file in the root directory) and Apache 2.0 License
   4 //  (found in the LICENSE.Apache file in the root directory).
   5 //
   6 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
   7 // Use of this source code is governed by a BSD-style license that can be
   8 // found in the LICENSE file. See the AUTHORS file for names of contributors.
   9
  10 #include "test_util/testutil.h"
  11
  12 #include <array>
  13 #include <cctype>
  14 #include <fstream>
  15 #include <sstream>
  16
  17 #include "db/memtable_list.h"
  18 #include "env/composite_env_wrapper.h"
  19 #include "file/random_access_file_reader.h"
  20 #include "file/sequence_file_reader.h"
  21 #include "file/writable_file_writer.h"
  22 #include "port/port.h"
  23
  24 namespace ROCKSDB_NAMESPACE {
  25 namespace test {
  26
  27 const uint32_t kDefaultFormatVersion = BlockBasedTableOptions().format_version;
  28 const uint32_t kLatestFormatVersion = 5u;
  29
  30 Slice RandomString(Random* rnd, int len, std::string* dst) {
  31   dst->resize(len);
  32   for (int i = 0; i < len; i++) {
  33     (*dst)[i] = static_cast<char>(' ' + rnd->Uniform(95));  // ' ' .. '~'
  34   }
  35   return Slice(*dst);
  36 }
  37
  38 extern std::string RandomHumanReadableString(Random* rnd, int len) {
  39   std::string ret;
  40   ret.resize(len);
  41   for (int i = 0; i < len; ++i) {
  42     ret[i] = static_cast<char>('a' + rnd->Uniform(26));
  43   }
  44   return ret;
  45 }
  46
  47 std::string RandomKey(Random* rnd, int len, RandomKeyType type) {
  48   // Make sure to generate a wide variety of characters so we
  49   // test the boundary conditions for short-key optimizations.
  50   static const char kTestChars[] = {'\0', '\1', 'a',    'b',    'c',
  51                                     'd',  'e',  '\xfd', '\xfe', '\xff'};
  52   std::string result;
  53   for (int i = 0; i < len; i++) {
  54     std::size_t indx = 0;
  55     switch (type) {
  56       case RandomKeyType::RANDOM:
  57         indx = rnd->Uniform(sizeof(kTestChars));
  58         break;
  59       case RandomKeyType::LARGEST:
  60         indx = sizeof(kTestChars) - 1;
  61         break;
  62       case RandomKeyType::MIDDLE:
  63         indx = sizeof(kTestChars) / 2;
  64         break;
  65       case RandomKeyType::SMALLEST:
  66         indx = 0;
  67         break;
  68     }
  69     result += kTestChars[indx];
  70   }
  71   return result;
  72 }
  73
  74 extern Slice CompressibleString(Random* rnd, double compressed_fraction,
  75                                 int len, std::string* dst) {
  76   int raw = static_cast<int>(len * compressed_fraction);
  77   if (raw < 1) raw = 1;
  78   std::string raw_data;
  79   RandomString(rnd, raw, &raw_data);
  80
  81   // Duplicate the random data until we have filled "len" bytes
  82   dst->clear();
  83   while (dst->size() < (unsigned int)len) {
  84     dst->append(raw_data);
  85   }
  86   dst->resize(len);
  87   return Slice(*dst);
  88 }
  89
  90 namespace {
  91 class Uint64ComparatorImpl : public Comparator {
  92  public:
  93   Uint64ComparatorImpl() {}
  94
  95   const char* Name() const override { return "rocksdb.Uint64Comparator"; }
  96
  97   int Compare(const Slice& a, const Slice& b) const override {
  98     assert(a.size() == sizeof(uint64_t) && b.size() == sizeof(uint64_t));
  99     const uint64_t* left = reinterpret_cast<const uint64_t*>(a.data());
 100     const uint64_t* right = reinterpret_cast<const uint64_t*>(b.data());
 101     uint64_t leftValue;
 102     uint64_t rightValue;
 103     GetUnaligned(left, &leftValue);
 104     GetUnaligned(right, &rightValue);
 105     if (leftValue == rightValue) {
 106       return 0;
 107     } else if (leftValue < rightValue) {
 108       return -1;
 109     } else {
 110       return 1;
 111     }
 112   }
 113
 114   void FindShortestSeparator(std::string* /*start*/,
 115                              const Slice& /*limit*/) const override {
 116     return;
 117   }
 118
 119   void FindShortSuccessor(std::string* /*key*/) const override { return; }
 120 };
 121 }  // namespace
 122
 123 const Comparator* Uint64Comparator() {
 124   static Uint64ComparatorImpl uint64comp;
 125   return &uint64comp;
 126 }
 127
 128 WritableFileWriter* GetWritableFileWriter(WritableFile* wf,
 129                                           const std::string& fname) {
 130   std::unique_ptr<WritableFile> file(wf);
 131   return new WritableFileWriter(NewLegacyWritableFileWrapper(std::move(file)),
 132                                 fname, EnvOptions());
 133 }
 134
 135 RandomAccessFileReader* GetRandomAccessFileReader(RandomAccessFile* raf) {
 136   std::unique_ptr<RandomAccessFile> file(raf);
 137   return new RandomAccessFileReader(NewLegacyRandomAccessFileWrapper(file),
 138                                     "[test RandomAccessFileReader]");
 139 }
 140
 141 SequentialFileReader* GetSequentialFileReader(SequentialFile* se,
 142                                               const std::string& fname) {
 143   std::unique_ptr<SequentialFile> file(se);
 144   return new SequentialFileReader(NewLegacySequentialFileWrapper(file), fname);
 145 }
 146
 147 void CorruptKeyType(InternalKey* ikey) {
 148   std::string keystr = ikey->Encode().ToString();
 149   keystr[keystr.size() - 8] = kTypeLogData;
 150   ikey->DecodeFrom(Slice(keystr.data(), keystr.size()));
 151 }
 152
 153 std::string KeyStr(const std::string& user_key, const SequenceNumber& seq,
 154                    const ValueType& t, bool corrupt) {
 155   InternalKey k(user_key, seq, t);
 156   if (corrupt) {
 157     CorruptKeyType(&k);
 158   }
 159   return k.Encode().ToString();
 160 }
 161
 162 std::string RandomName(Random* rnd, const size_t len) {
 163   std::stringstream ss;
 164   for (size_t i = 0; i < len; ++i) {
 165     ss << static_cast<char>(rnd->Uniform(26) + 'a');
 166   }
 167   return ss.str();
 168 }
 169
 170 CompressionType RandomCompressionType(Random* rnd) {
 171   auto ret = static_cast<CompressionType>(rnd->Uniform(6));
 172   while (!CompressionTypeSupported(ret)) {
 173     ret = static_cast<CompressionType>((static_cast<int>(ret) + 1) % 6);
 174   }
 175   return ret;
 176 }
 177
 178 void RandomCompressionTypeVector(const size_t count,
 179                                  std::vector<CompressionType>* types,
 180                                  Random* rnd) {
 181   types->clear();
 182   for (size_t i = 0; i < count; ++i) {
 183     types->emplace_back(RandomCompressionType(rnd));
 184   }
 185 }
 186
 187 const SliceTransform* RandomSliceTransform(Random* rnd, int pre_defined) {
 188   int random_num = pre_defined >= 0 ? pre_defined : rnd->Uniform(4);
 189   switch (random_num) {
 190     case 0:
 191       return NewFixedPrefixTransform(rnd->Uniform(20) + 1);
 192     case 1:
 193       return NewCappedPrefixTransform(rnd->Uniform(20) + 1);
 194     case 2:
 195       return NewNoopTransform();
 196     default:
 197       return nullptr;
 198   }
 199 }
 200
 201 BlockBasedTableOptions RandomBlockBasedTableOptions(Random* rnd) {
 202   BlockBasedTableOptions opt;
 203   opt.cache_index_and_filter_blocks = rnd->Uniform(2);
 204   opt.pin_l0_filter_and_index_blocks_in_cache = rnd->Uniform(2);
 205   opt.pin_top_level_index_and_filter = rnd->Uniform(2);
 206   using IndexType = BlockBasedTableOptions::IndexType;
 207   const std::array<IndexType, 4> index_types = {
 208       {IndexType::kBinarySearch, IndexType::kHashSearch,
 209        IndexType::kTwoLevelIndexSearch, IndexType::kBinarySearchWithFirstKey}};
 210   opt.index_type =
 211       index_types[rnd->Uniform(static_cast<int>(index_types.size()))];
 212   opt.hash_index_allow_collision = rnd->Uniform(2);
 213   opt.checksum = static_cast<ChecksumType>(rnd->Uniform(3));
 214   opt.block_size = rnd->Uniform(10000000);
 215   opt.block_size_deviation = rnd->Uniform(100);
 216   opt.block_restart_interval = rnd->Uniform(100);
 217   opt.index_block_restart_interval = rnd->Uniform(100);
 218   opt.whole_key_filtering = rnd->Uniform(2);
 219
 220   return opt;
 221 }
 222
 223 TableFactory* RandomTableFactory(Random* rnd, int pre_defined) {
 224 #ifndef ROCKSDB_LITE
 225   int random_num = pre_defined >= 0 ? pre_defined : rnd->Uniform(4);
 226   switch (random_num) {
 227     case 0:
 228       return NewPlainTableFactory();
 229     case 1:
 230       return NewCuckooTableFactory();
 231     default:
 232       return NewBlockBasedTableFactory();
 233   }
 234 #else
 235   (void)rnd;
 236   (void)pre_defined;
 237   return NewBlockBasedTableFactory();
 238 #endif  // !ROCKSDB_LITE
 239 }
 240
 241 MergeOperator* RandomMergeOperator(Random* rnd) {
 242   return new ChanglingMergeOperator(RandomName(rnd, 10));
 243 }
 244
 245 CompactionFilter* RandomCompactionFilter(Random* rnd) {
 246   return new ChanglingCompactionFilter(RandomName(rnd, 10));
 247 }
 248
 249 CompactionFilterFactory* RandomCompactionFilterFactory(Random* rnd) {
 250   return new ChanglingCompactionFilterFactory(RandomName(rnd, 10));
 251 }
 252
 253 void RandomInitDBOptions(DBOptions* db_opt, Random* rnd) {
 254   // boolean options
 255   db_opt->advise_random_on_open = rnd->Uniform(2);
 256   db_opt->allow_mmap_reads = rnd->Uniform(2);
 257   db_opt->allow_mmap_writes = rnd->Uniform(2);
 258   db_opt->use_direct_reads = rnd->Uniform(2);
 259   db_opt->use_direct_io_for_flush_and_compaction = rnd->Uniform(2);
 260   db_opt->create_if_missing = rnd->Uniform(2);
 261   db_opt->create_missing_column_families = rnd->Uniform(2);
 262   db_opt->enable_thread_tracking = rnd->Uniform(2);
 263   db_opt->error_if_exists = rnd->Uniform(2);
 264   db_opt->is_fd_close_on_exec = rnd->Uniform(2);
 265   db_opt->paranoid_checks = rnd->Uniform(2);
 266   db_opt->skip_log_error_on_recovery = rnd->Uniform(2);
 267   db_opt->skip_stats_update_on_db_open = rnd->Uniform(2);
 268   db_opt->skip_checking_sst_file_sizes_on_db_open = rnd->Uniform(2);
 269   db_opt->use_adaptive_mutex = rnd->Uniform(2);
 270   db_opt->use_fsync = rnd->Uniform(2);
 271   db_opt->recycle_log_file_num = rnd->Uniform(2);
 272   db_opt->avoid_flush_during_recovery = rnd->Uniform(2);
 273   db_opt->avoid_flush_during_shutdown = rnd->Uniform(2);
 274
 275   // int options
 276   db_opt->max_background_compactions = rnd->Uniform(100);
 277   db_opt->max_background_flushes = rnd->Uniform(100);
 278   db_opt->max_file_opening_threads = rnd->Uniform(100);
 279   db_opt->max_open_files = rnd->Uniform(100);
 280   db_opt->table_cache_numshardbits = rnd->Uniform(100);
 281
 282   // size_t options
 283   db_opt->db_write_buffer_size = rnd->Uniform(10000);
 284   db_opt->keep_log_file_num = rnd->Uniform(10000);
 285   db_opt->log_file_time_to_roll = rnd->Uniform(10000);
 286   db_opt->manifest_preallocation_size = rnd->Uniform(10000);
 287   db_opt->max_log_file_size = rnd->Uniform(10000);
 288
 289   // std::string options
 290   db_opt->db_log_dir = "path/to/db_log_dir";
 291   db_opt->wal_dir = "path/to/wal_dir";
 292
 293   // uint32_t options
 294   db_opt->max_subcompactions = rnd->Uniform(100000);
 295
 296   // uint64_t options
 297   static const uint64_t uint_max = static_cast<uint64_t>(UINT_MAX);
 298   db_opt->WAL_size_limit_MB = uint_max + rnd->Uniform(100000);
 299   db_opt->WAL_ttl_seconds = uint_max + rnd->Uniform(100000);
 300   db_opt->bytes_per_sync = uint_max + rnd->Uniform(100000);
 301   db_opt->delayed_write_rate = uint_max + rnd->Uniform(100000);
 302   db_opt->delete_obsolete_files_period_micros = uint_max + rnd->Uniform(100000);
 303   db_opt->max_manifest_file_size = uint_max + rnd->Uniform(100000);
 304   db_opt->max_total_wal_size = uint_max + rnd->Uniform(100000);
 305   db_opt->wal_bytes_per_sync = uint_max + rnd->Uniform(100000);
 306
 307   // unsigned int options
 308   db_opt->stats_dump_period_sec = rnd->Uniform(100000);
 309 }
 310
 311 void RandomInitCFOptions(ColumnFamilyOptions* cf_opt, DBOptions& db_options,
 312                          Random* rnd) {
 313   cf_opt->compaction_style = (CompactionStyle)(rnd->Uniform(4));
 314
 315   // boolean options
 316   cf_opt->report_bg_io_stats = rnd->Uniform(2);
 317   cf_opt->disable_auto_compactions = rnd->Uniform(2);
 318   cf_opt->inplace_update_support = rnd->Uniform(2);
 319   cf_opt->level_compaction_dynamic_level_bytes = rnd->Uniform(2);
 320   cf_opt->optimize_filters_for_hits = rnd->Uniform(2);
 321   cf_opt->paranoid_file_checks = rnd->Uniform(2);
 322   cf_opt->purge_redundant_kvs_while_flush = rnd->Uniform(2);
 323   cf_opt->force_consistency_checks = rnd->Uniform(2);
 324   cf_opt->compaction_options_fifo.allow_compaction = rnd->Uniform(2);
 325   cf_opt->memtable_whole_key_filtering = rnd->Uniform(2);
 326
 327   // double options
 328   cf_opt->hard_rate_limit = static_cast<double>(rnd->Uniform(10000)) / 13;
 329   cf_opt->soft_rate_limit = static_cast<double>(rnd->Uniform(10000)) / 13;
 330   cf_opt->memtable_prefix_bloom_size_ratio =
 331       static_cast<double>(rnd->Uniform(10000)) / 20000.0;
 332
 333   // int options
 334   cf_opt->level0_file_num_compaction_trigger = rnd->Uniform(100);
 335   cf_opt->level0_slowdown_writes_trigger = rnd->Uniform(100);
 336   cf_opt->level0_stop_writes_trigger = rnd->Uniform(100);
 337   cf_opt->max_bytes_for_level_multiplier = rnd->Uniform(100);
 338   cf_opt->max_mem_compaction_level = rnd->Uniform(100);
 339   cf_opt->max_write_buffer_number = rnd->Uniform(100);
 340   cf_opt->max_write_buffer_number_to_maintain = rnd->Uniform(100);
 341   cf_opt->max_write_buffer_size_to_maintain = rnd->Uniform(10000);
 342   cf_opt->min_write_buffer_number_to_merge = rnd->Uniform(100);
 343   cf_opt->num_levels = rnd->Uniform(100);
 344   cf_opt->target_file_size_multiplier = rnd->Uniform(100);
 345
 346   // vector int options
 347   cf_opt->max_bytes_for_level_multiplier_additional.resize(cf_opt->num_levels);
 348   for (int i = 0; i < cf_opt->num_levels; i++) {
 349     cf_opt->max_bytes_for_level_multiplier_additional[i] = rnd->Uniform(100);
 350   }
 351
 352   // size_t options
 353   cf_opt->arena_block_size = rnd->Uniform(10000);
 354   cf_opt->inplace_update_num_locks = rnd->Uniform(10000);
 355   cf_opt->max_successive_merges = rnd->Uniform(10000);
 356   cf_opt->memtable_huge_page_size = rnd->Uniform(10000);
 357   cf_opt->write_buffer_size = rnd->Uniform(10000);
 358
 359   // uint32_t options
 360   cf_opt->bloom_locality = rnd->Uniform(10000);
 361   cf_opt->max_bytes_for_level_base = rnd->Uniform(10000);
 362
 363   // uint64_t options
 364   static const uint64_t uint_max = static_cast<uint64_t>(UINT_MAX);
 365   cf_opt->ttl =
 366       db_options.max_open_files == -1 ? uint_max + rnd->Uniform(10000) : 0;
 367   cf_opt->periodic_compaction_seconds =
 368       db_options.max_open_files == -1 ? uint_max + rnd->Uniform(10000) : 0;
 369   cf_opt->max_sequential_skip_in_iterations = uint_max + rnd->Uniform(10000);
 370   cf_opt->target_file_size_base = uint_max + rnd->Uniform(10000);
 371   cf_opt->max_compaction_bytes =
 372       cf_opt->target_file_size_base * rnd->Uniform(100);
 373   cf_opt->compaction_options_fifo.max_table_files_size =
 374       uint_max + rnd->Uniform(10000);
 375
 376   // unsigned int options
 377   cf_opt->rate_limit_delay_max_milliseconds = rnd->Uniform(10000);
 378
 379   // pointer typed options
 380   cf_opt->prefix_extractor.reset(RandomSliceTransform(rnd));
 381   cf_opt->table_factory.reset(RandomTableFactory(rnd));
 382   cf_opt->merge_operator.reset(RandomMergeOperator(rnd));
 383   if (cf_opt->compaction_filter) {
 384     delete cf_opt->compaction_filter;
 385   }
 386   cf_opt->compaction_filter = RandomCompactionFilter(rnd);
 387   cf_opt->compaction_filter_factory.reset(RandomCompactionFilterFactory(rnd));
 388
 389   // custom typed options
 390   cf_opt->compression = RandomCompressionType(rnd);
 391   RandomCompressionTypeVector(cf_opt->num_levels,
 392                               &cf_opt->compression_per_level, rnd);
 393 }
 394
 395 Status DestroyDir(Env* env, const std::string& dir) {
 396   Status s;
 397   if (env->FileExists(dir).IsNotFound()) {
 398     return s;
 399   }
 400   std::vector<std::string> files_in_dir;
 401   s = env->GetChildren(dir, &files_in_dir);
 402   if (s.ok()) {
 403     for (auto& file_in_dir : files_in_dir) {
 404       if (file_in_dir == "." || file_in_dir == "..") {
 405         continue;
 406       }
 407       s = env->DeleteFile(dir + "/" + file_in_dir);
 408       if (!s.ok()) {
 409         break;
 410       }
 411     }
 412   }
 413
 414   if (s.ok()) {
 415     s = env->DeleteDir(dir);
 416   }
 417   return s;
 418 }
 419
 420 bool IsDirectIOSupported(Env* env, const std::string& dir) {
 421   EnvOptions env_options;
 422   env_options.use_mmap_writes = false;
 423   env_options.use_direct_writes = true;
 424   std::string tmp = TempFileName(dir, 999);
 425   Status s;
 426   {
 427     std::unique_ptr<WritableFile> file;
 428     s = env->NewWritableFile(tmp, &file, env_options);
 429   }
 430   if (s.ok()) {
 431     s = env->DeleteFile(tmp);
 432   }
 433   return s.ok();
 434 }
 435
 436 size_t GetLinesCount(const std::string& fname, const std::string& pattern) {
 437   std::stringstream ssbuf;
 438   std::string line;
 439   size_t count = 0;
 440
 441   std::ifstream inFile(fname.c_str());
 442   ssbuf << inFile.rdbuf();
 443
 444   while (getline(ssbuf, line)) {
 445     if (line.find(pattern) != std::string::npos) {
 446       count++;
 447     }
 448   }
 449
 450   return count;
 451 }
 452
 453 }  // namespace test
 454 }  // namespace ROCKSDB_NAMESPACE