ceph/src/rocksdb/util/testutil.cc

   1 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
   2 //  This source code is licensed under the BSD-style license found in the
   3 //  LICENSE file in the root directory of this source tree. An additional grant
   4 //  of patent rights can be found in the PATENTS file in the same directory.
   5 //
   6 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
   7 // Use of this source code is governed by a BSD-style license that can be
   8 // found in the LICENSE file. See the AUTHORS file for names of contributors.
   9
  10 #include "util/testutil.h"
  11
  12 #include <cctype>
  13 #include <sstream>
  14
  15 #include "db/memtable_list.h"
  16 #include "port/port.h"
  17 #include "util/file_reader_writer.h"
  18
  19 namespace rocksdb {
  20 namespace test {
  21
  22 Slice RandomString(Random* rnd, int len, std::string* dst) {
  23   dst->resize(len);
  24   for (int i = 0; i < len; i++) {
  25     (*dst)[i] = static_cast<char>(' ' + rnd->Uniform(95));   // ' ' .. '~'
  26   }
  27   return Slice(*dst);
  28 }
  29
  30 extern std::string RandomHumanReadableString(Random* rnd, int len) {
  31   std::string ret;
  32   ret.resize(len);
  33   for (int i = 0; i < len; ++i) {
  34     ret[i] = static_cast<char>('a' + rnd->Uniform(26));
  35   }
  36   return ret;
  37 }
  38
  39 std::string RandomKey(Random* rnd, int len, RandomKeyType type) {
  40   // Make sure to generate a wide variety of characters so we
  41   // test the boundary conditions for short-key optimizations.
  42   static const char kTestChars[] = {
  43     '\0', '\1', 'a', 'b', 'c', 'd', 'e', '\xfd', '\xfe', '\xff'
  44   };
  45   std::string result;
  46   for (int i = 0; i < len; i++) {
  47     std::size_t indx = 0;
  48     switch (type) {
  49       case RandomKeyType::RANDOM:
  50         indx = rnd->Uniform(sizeof(kTestChars));
  51         break;
  52       case RandomKeyType::LARGEST:
  53         indx = sizeof(kTestChars) - 1;
  54         break;
  55       case RandomKeyType::MIDDLE:
  56         indx = sizeof(kTestChars) / 2;
  57         break;
  58       case RandomKeyType::SMALLEST:
  59         indx = 0;
  60         break;
  61     }
  62     result += kTestChars[indx];
  63   }
  64   return result;
  65 }
  66
  67
  68 extern Slice CompressibleString(Random* rnd, double compressed_fraction,
  69                                 int len, std::string* dst) {
  70   int raw = static_cast<int>(len * compressed_fraction);
  71   if (raw < 1) raw = 1;
  72   std::string raw_data;
  73   RandomString(rnd, raw, &raw_data);
  74
  75   // Duplicate the random data until we have filled "len" bytes
  76   dst->clear();
  77   while (dst->size() < (unsigned int)len) {
  78     dst->append(raw_data);
  79   }
  80   dst->resize(len);
  81   return Slice(*dst);
  82 }
  83
  84 namespace {
  85 class Uint64ComparatorImpl : public Comparator {
  86  public:
  87   Uint64ComparatorImpl() { }
  88
  89   virtual const char* Name() const override {
  90     return "rocksdb.Uint64Comparator";
  91   }
  92
  93   virtual int Compare(const Slice& a, const Slice& b) const override {
  94     assert(a.size() == sizeof(uint64_t) && b.size() == sizeof(uint64_t));
  95     const uint64_t* left = reinterpret_cast<const uint64_t*>(a.data());
  96     const uint64_t* right = reinterpret_cast<const uint64_t*>(b.data());
  97     uint64_t leftValue;
  98     uint64_t rightValue;
  99     GetUnaligned(left, &leftValue);
 100     GetUnaligned(right, &rightValue);
 101     if (leftValue == rightValue) {
 102       return 0;
 103     } else if (leftValue < rightValue) {
 104       return -1;
 105     } else {
 106       return 1;
 107     }
 108   }
 109
 110   virtual void FindShortestSeparator(std::string* start,
 111       const Slice& limit) const override {
 112     return;
 113   }
 114
 115   virtual void FindShortSuccessor(std::string* key) const override {
 116     return;
 117   }
 118 };
 119 }  // namespace
 120
 121 static port::OnceType once;
 122 static const Comparator* uint64comp;
 123
 124 static void InitModule() {
 125   uint64comp = new Uint64ComparatorImpl;
 126 }
 127
 128 const Comparator* Uint64Comparator() {
 129   port::InitOnce(&once, InitModule);
 130   return uint64comp;
 131 }
 132
 133 WritableFileWriter* GetWritableFileWriter(WritableFile* wf) {
 134   unique_ptr<WritableFile> file(wf);
 135   return new WritableFileWriter(std::move(file), EnvOptions());
 136 }
 137
 138 RandomAccessFileReader* GetRandomAccessFileReader(RandomAccessFile* raf) {
 139   unique_ptr<RandomAccessFile> file(raf);
 140   return new RandomAccessFileReader(std::move(file));
 141 }
 142
 143 SequentialFileReader* GetSequentialFileReader(SequentialFile* se) {
 144   unique_ptr<SequentialFile> file(se);
 145   return new SequentialFileReader(std::move(file));
 146 }
 147
 148 void CorruptKeyType(InternalKey* ikey) {
 149   std::string keystr = ikey->Encode().ToString();
 150   keystr[keystr.size() - 8] = kTypeLogData;
 151   ikey->DecodeFrom(Slice(keystr.data(), keystr.size()));
 152 }
 153
 154 std::string KeyStr(const std::string& user_key, const SequenceNumber& seq,
 155                    const ValueType& t, bool corrupt) {
 156   InternalKey k(user_key, seq, t);
 157   if (corrupt) {
 158     CorruptKeyType(&k);
 159   }
 160   return k.Encode().ToString();
 161 }
 162
 163 std::string RandomName(Random* rnd, const size_t len) {
 164   std::stringstream ss;
 165   for (size_t i = 0; i < len; ++i) {
 166     ss << static_cast<char>(rnd->Uniform(26) + 'a');
 167   }
 168   return ss.str();
 169 }
 170
 171 CompressionType RandomCompressionType(Random* rnd) {
 172   return static_cast<CompressionType>(rnd->Uniform(6));
 173 }
 174
 175 void RandomCompressionTypeVector(const size_t count,
 176                                  std::vector<CompressionType>* types,
 177                                  Random* rnd) {
 178   types->clear();
 179   for (size_t i = 0; i < count; ++i) {
 180     types->emplace_back(RandomCompressionType(rnd));
 181   }
 182 }
 183
 184 const SliceTransform* RandomSliceTransform(Random* rnd, int pre_defined) {
 185   int random_num = pre_defined >= 0 ? pre_defined : rnd->Uniform(4);
 186   switch (random_num) {
 187     case 0:
 188       return NewFixedPrefixTransform(rnd->Uniform(20) + 1);
 189     case 1:
 190       return NewCappedPrefixTransform(rnd->Uniform(20) + 1);
 191     case 2:
 192       return NewNoopTransform();
 193     default:
 194       return nullptr;
 195   }
 196 }
 197
 198 BlockBasedTableOptions RandomBlockBasedTableOptions(Random* rnd) {
 199   BlockBasedTableOptions opt;
 200   opt.cache_index_and_filter_blocks = rnd->Uniform(2);
 201   opt.pin_l0_filter_and_index_blocks_in_cache = rnd->Uniform(2);
 202   opt.index_type = rnd->Uniform(2) ? BlockBasedTableOptions::kBinarySearch
 203                                    : BlockBasedTableOptions::kHashSearch;
 204   opt.hash_index_allow_collision = rnd->Uniform(2);
 205   opt.checksum = static_cast<ChecksumType>(rnd->Uniform(3));
 206   opt.block_size = rnd->Uniform(10000000);
 207   opt.block_size_deviation = rnd->Uniform(100);
 208   opt.block_restart_interval = rnd->Uniform(100);
 209   opt.index_block_restart_interval = rnd->Uniform(100);
 210   opt.whole_key_filtering = rnd->Uniform(2);
 211
 212   return opt;
 213 }
 214
 215 TableFactory* RandomTableFactory(Random* rnd, int pre_defined) {
 216 #ifndef ROCKSDB_LITE
 217   int random_num = pre_defined >= 0 ? pre_defined : rnd->Uniform(4);
 218   switch (random_num) {
 219     case 0:
 220       return NewPlainTableFactory();
 221     case 1:
 222       return NewCuckooTableFactory();
 223     default:
 224       return NewBlockBasedTableFactory();
 225   }
 226 #else
 227   return NewBlockBasedTableFactory();
 228 #endif  // !ROCKSDB_LITE
 229 }
 230
 231 MergeOperator* RandomMergeOperator(Random* rnd) {
 232   return new ChanglingMergeOperator(RandomName(rnd, 10));
 233 }
 234
 235 CompactionFilter* RandomCompactionFilter(Random* rnd) {
 236   return new ChanglingCompactionFilter(RandomName(rnd, 10));
 237 }
 238
 239 CompactionFilterFactory* RandomCompactionFilterFactory(Random* rnd) {
 240   return new ChanglingCompactionFilterFactory(RandomName(rnd, 10));
 241 }
 242
 243 void RandomInitDBOptions(DBOptions* db_opt, Random* rnd) {
 244   // boolean options
 245   db_opt->advise_random_on_open = rnd->Uniform(2);
 246   db_opt->allow_mmap_reads = rnd->Uniform(2);
 247   db_opt->allow_mmap_writes = rnd->Uniform(2);
 248   db_opt->use_direct_reads = rnd->Uniform(2);
 249   db_opt->use_direct_io_for_flush_and_compaction = rnd->Uniform(2);
 250   db_opt->create_if_missing = rnd->Uniform(2);
 251   db_opt->create_missing_column_families = rnd->Uniform(2);
 252   db_opt->enable_thread_tracking = rnd->Uniform(2);
 253   db_opt->error_if_exists = rnd->Uniform(2);
 254   db_opt->is_fd_close_on_exec = rnd->Uniform(2);
 255   db_opt->paranoid_checks = rnd->Uniform(2);
 256   db_opt->skip_log_error_on_recovery = rnd->Uniform(2);
 257   db_opt->skip_stats_update_on_db_open = rnd->Uniform(2);
 258   db_opt->use_adaptive_mutex = rnd->Uniform(2);
 259   db_opt->use_fsync = rnd->Uniform(2);
 260   db_opt->recycle_log_file_num = rnd->Uniform(2);
 261   db_opt->avoid_flush_during_recovery = rnd->Uniform(2);
 262   db_opt->avoid_flush_during_shutdown = rnd->Uniform(2);
 263
 264   // int options
 265   db_opt->max_background_compactions = rnd->Uniform(100);
 266   db_opt->max_background_flushes = rnd->Uniform(100);
 267   db_opt->max_file_opening_threads = rnd->Uniform(100);
 268   db_opt->max_open_files = rnd->Uniform(100);
 269   db_opt->table_cache_numshardbits = rnd->Uniform(100);
 270
 271   // size_t options
 272   db_opt->db_write_buffer_size = rnd->Uniform(10000);
 273   db_opt->keep_log_file_num = rnd->Uniform(10000);
 274   db_opt->log_file_time_to_roll = rnd->Uniform(10000);
 275   db_opt->manifest_preallocation_size = rnd->Uniform(10000);
 276   db_opt->max_log_file_size = rnd->Uniform(10000);
 277
 278   // std::string options
 279   db_opt->db_log_dir = "path/to/db_log_dir";
 280   db_opt->wal_dir = "path/to/wal_dir";
 281
 282   // uint32_t options
 283   db_opt->max_subcompactions = rnd->Uniform(100000);
 284
 285   // uint64_t options
 286   static const uint64_t uint_max = static_cast<uint64_t>(UINT_MAX);
 287   db_opt->WAL_size_limit_MB = uint_max + rnd->Uniform(100000);
 288   db_opt->WAL_ttl_seconds = uint_max + rnd->Uniform(100000);
 289   db_opt->bytes_per_sync = uint_max + rnd->Uniform(100000);
 290   db_opt->delayed_write_rate = uint_max + rnd->Uniform(100000);
 291   db_opt->delete_obsolete_files_period_micros = uint_max + rnd->Uniform(100000);
 292   db_opt->max_manifest_file_size = uint_max + rnd->Uniform(100000);
 293   db_opt->max_total_wal_size = uint_max + rnd->Uniform(100000);
 294   db_opt->wal_bytes_per_sync = uint_max + rnd->Uniform(100000);
 295
 296   // unsigned int options
 297   db_opt->stats_dump_period_sec = rnd->Uniform(100000);
 298 }
 299
 300 void RandomInitCFOptions(ColumnFamilyOptions* cf_opt, Random* rnd) {
 301   cf_opt->compaction_style = (CompactionStyle)(rnd->Uniform(4));
 302
 303   // boolean options
 304   cf_opt->report_bg_io_stats = rnd->Uniform(2);
 305   cf_opt->disable_auto_compactions = rnd->Uniform(2);
 306   cf_opt->inplace_update_support = rnd->Uniform(2);
 307   cf_opt->level_compaction_dynamic_level_bytes = rnd->Uniform(2);
 308   cf_opt->optimize_filters_for_hits = rnd->Uniform(2);
 309   cf_opt->paranoid_file_checks = rnd->Uniform(2);
 310   cf_opt->purge_redundant_kvs_while_flush = rnd->Uniform(2);
 311   cf_opt->force_consistency_checks = rnd->Uniform(2);
 312
 313   // double options
 314   cf_opt->hard_rate_limit = static_cast<double>(rnd->Uniform(10000)) / 13;
 315   cf_opt->soft_rate_limit = static_cast<double>(rnd->Uniform(10000)) / 13;
 316   cf_opt->memtable_prefix_bloom_size_ratio =
 317       static_cast<double>(rnd->Uniform(10000)) / 20000.0;
 318
 319   // int options
 320   cf_opt->level0_file_num_compaction_trigger = rnd->Uniform(100);
 321   cf_opt->level0_slowdown_writes_trigger = rnd->Uniform(100);
 322   cf_opt->level0_stop_writes_trigger = rnd->Uniform(100);
 323   cf_opt->max_bytes_for_level_multiplier = rnd->Uniform(100);
 324   cf_opt->max_mem_compaction_level = rnd->Uniform(100);
 325   cf_opt->max_write_buffer_number = rnd->Uniform(100);
 326   cf_opt->max_write_buffer_number_to_maintain = rnd->Uniform(100);
 327   cf_opt->min_write_buffer_number_to_merge = rnd->Uniform(100);
 328   cf_opt->num_levels = rnd->Uniform(100);
 329   cf_opt->target_file_size_multiplier = rnd->Uniform(100);
 330
 331   // vector int options
 332   cf_opt->max_bytes_for_level_multiplier_additional.resize(cf_opt->num_levels);
 333   for (int i = 0; i < cf_opt->num_levels; i++) {
 334     cf_opt->max_bytes_for_level_multiplier_additional[i] = rnd->Uniform(100);
 335   }
 336
 337   // size_t options
 338   cf_opt->arena_block_size = rnd->Uniform(10000);
 339   cf_opt->inplace_update_num_locks = rnd->Uniform(10000);
 340   cf_opt->max_successive_merges = rnd->Uniform(10000);
 341   cf_opt->memtable_huge_page_size = rnd->Uniform(10000);
 342   cf_opt->write_buffer_size = rnd->Uniform(10000);
 343
 344   // uint32_t options
 345   cf_opt->bloom_locality = rnd->Uniform(10000);
 346   cf_opt->max_bytes_for_level_base = rnd->Uniform(10000);
 347
 348   // uint64_t options
 349   static const uint64_t uint_max = static_cast<uint64_t>(UINT_MAX);
 350   cf_opt->max_sequential_skip_in_iterations = uint_max + rnd->Uniform(10000);
 351   cf_opt->target_file_size_base = uint_max + rnd->Uniform(10000);
 352   cf_opt->max_compaction_bytes =
 353       cf_opt->target_file_size_base * rnd->Uniform(100);
 354
 355   // unsigned int options
 356   cf_opt->rate_limit_delay_max_milliseconds = rnd->Uniform(10000);
 357
 358   // pointer typed options
 359   cf_opt->prefix_extractor.reset(RandomSliceTransform(rnd));
 360   cf_opt->table_factory.reset(RandomTableFactory(rnd));
 361   cf_opt->merge_operator.reset(RandomMergeOperator(rnd));
 362   if (cf_opt->compaction_filter) {
 363     delete cf_opt->compaction_filter;
 364   }
 365   cf_opt->compaction_filter = RandomCompactionFilter(rnd);
 366   cf_opt->compaction_filter_factory.reset(RandomCompactionFilterFactory(rnd));
 367
 368   // custom typed options
 369   cf_opt->compression = RandomCompressionType(rnd);
 370   RandomCompressionTypeVector(cf_opt->num_levels,
 371                               &cf_opt->compression_per_level, rnd);
 372 }
 373
 374 Status DestroyDir(Env* env, const std::string& dir) {
 375   Status s;
 376   if (env->FileExists(dir).IsNotFound()) {
 377     return s;
 378   }
 379   std::vector<std::string> files_in_dir;
 380   s = env->GetChildren(dir, &files_in_dir);
 381   if (s.ok()) {
 382     for (auto& file_in_dir : files_in_dir) {
 383       if (file_in_dir == "." || file_in_dir == "..") {
 384         continue;
 385       }
 386       s = env->DeleteFile(dir + "/" + file_in_dir);
 387       if (!s.ok()) {
 388         break;
 389       }
 390     }
 391   }
 392
 393   if (s.ok()) {
 394     s = env->DeleteDir(dir);
 395   }
 396   return s;
 397 }
 398
 399 }  // namespace test
 400 }  // namespace rocksdb