#include "db_stress_tool/db_stress_listener.h"
#include "db_stress_tool/db_stress_shared_state.h"
#include "db_stress_tool/db_stress_test_base.h"
-#include "hdfs/env_hdfs.h"
#include "logging/logging.h"
#include "monitoring/histogram.h"
#include "options/options_helper.h"
#include "rocksdb/slice.h"
#include "rocksdb/slice_transform.h"
#include "rocksdb/statistics.h"
-#include "rocksdb/utilities/backupable_db.h"
+#include "rocksdb/utilities/backup_engine.h"
#include "rocksdb/utilities/checkpoint.h"
#include "rocksdb/utilities/db_ttl.h"
#include "rocksdb/utilities/debug.h"
#include "util/random.h"
#include "util/string_util.h"
#include "utilities/blob_db/blob_db.h"
+#include "utilities/fault_injection_fs.h"
#include "utilities/merge_operators.h"
using GFLAGS_NAMESPACE::ParseCommandLineFlags;
DECLARE_int64(active_width);
DECLARE_bool(test_batches_snapshots);
DECLARE_bool(atomic_flush);
+DECLARE_int32(manual_wal_flush_one_in);
DECLARE_bool(test_cf_consistency);
+DECLARE_bool(test_multi_ops_txns);
DECLARE_int32(threads);
DECLARE_int32(ttl);
DECLARE_int32(value_size_mult);
DECLARE_bool(memtable_whole_key_filtering);
DECLARE_int32(open_files);
DECLARE_int64(compressed_cache_size);
+DECLARE_int32(compressed_cache_numshardbits);
DECLARE_int32(compaction_style);
+DECLARE_int32(compaction_pri);
DECLARE_int32(num_levels);
DECLARE_int32(level0_file_num_compaction_trigger);
DECLARE_int32(level0_slowdown_writes_trigger);
DECLARE_int32(block_size);
DECLARE_int32(format_version);
DECLARE_int32(index_block_restart_interval);
+DECLARE_bool(disable_auto_compactions);
DECLARE_int32(max_background_compactions);
DECLARE_int32(num_bottom_pri_threads);
DECLARE_int32(compaction_thread_pool_adjust_interval);
DECLARE_int32(set_options_one_in);
DECLARE_int32(set_in_place_one_in);
DECLARE_int64(cache_size);
+DECLARE_int32(cache_numshardbits);
DECLARE_bool(cache_index_and_filter_blocks);
+DECLARE_bool(charge_compression_dictionary_building_buffer);
+DECLARE_bool(charge_filter_construction);
+DECLARE_bool(charge_table_reader);
+DECLARE_bool(charge_file_metadata);
+DECLARE_bool(charge_blob_cache);
DECLARE_int32(top_level_index_pinning);
DECLARE_int32(partition_pinning);
DECLARE_int32(unpartitioned_pinning);
-DECLARE_bool(use_clock_cache);
+DECLARE_string(cache_type);
DECLARE_uint64(subcompactions);
DECLARE_uint64(periodic_compaction_seconds);
DECLARE_uint64(compaction_ttl);
DECLARE_bool(allow_concurrent_memtable_write);
+DECLARE_double(experimental_mempurge_threshold);
DECLARE_bool(enable_write_thread_adaptive_yield);
DECLARE_int32(reopen);
DECLARE_double(bloom_bits);
-DECLARE_bool(use_block_based_filter);
-DECLARE_bool(use_ribbon_filter);
+DECLARE_int32(ribbon_starting_level);
DECLARE_bool(partition_filters);
DECLARE_bool(optimize_filters_for_memory);
+DECLARE_bool(detect_filter_construct_corruption);
DECLARE_int32(index_type);
+DECLARE_int32(data_block_index_type);
DECLARE_string(db);
DECLARE_string(secondaries_base);
DECLARE_bool(test_secondary);
-DECLARE_string(expected_values_path);
+DECLARE_string(expected_values_dir);
DECLARE_bool(verify_checksum);
DECLARE_bool(mmap_read);
DECLARE_bool(mmap_write);
DECLARE_bool(statistics);
DECLARE_bool(sync);
DECLARE_bool(use_fsync);
+DECLARE_uint64(stats_dump_period_sec);
+DECLARE_uint64(bytes_per_sync);
+DECLARE_uint64(wal_bytes_per_sync);
DECLARE_int32(kill_random_test);
DECLARE_string(kill_exclude_prefixes);
DECLARE_bool(disable_wal);
DECLARE_int32(range_deletion_width);
DECLARE_uint64(rate_limiter_bytes_per_sec);
DECLARE_bool(rate_limit_bg_reads);
+DECLARE_bool(rate_limit_user_ops);
+DECLARE_bool(rate_limit_auto_wal_flush);
DECLARE_uint64(sst_file_manager_bytes_per_sec);
DECLARE_uint64(sst_file_manager_bytes_per_truncate);
DECLARE_bool(use_txn);
DECLARE_int32(nooverwritepercent);
DECLARE_int32(iterpercent);
DECLARE_uint64(num_iterations);
+DECLARE_int32(customopspercent);
DECLARE_string(compression_type);
DECLARE_string(bottommost_compression_type);
DECLARE_int32(compression_max_dict_bytes);
DECLARE_int32(compression_zstd_max_train_bytes);
DECLARE_int32(compression_parallel_threads);
+DECLARE_uint64(compression_max_dict_buffer_bytes);
+DECLARE_bool(compression_use_zstd_dict_trainer);
DECLARE_string(checksum_type);
-DECLARE_string(hdfs);
DECLARE_string(env_uri);
DECLARE_string(fs_uri);
DECLARE_uint64(ops_per_thread);
DECLARE_uint64(log2_keys_per_lock);
DECLARE_uint64(max_manifest_file_size);
DECLARE_bool(in_place_update);
-DECLARE_int32(secondary_catch_up_one_in);
DECLARE_string(memtablerep);
DECLARE_int32(prefix_size);
DECLARE_bool(use_merge);
+DECLARE_uint32(use_put_entity_one_in);
DECLARE_bool(use_full_merge_v1);
DECLARE_int32(sync_wal_one_in);
DECLARE_bool(avoid_unnecessary_blocking_io);
DECLARE_string(file_checksum_impl);
#ifndef ROCKSDB_LITE
+// Options for StackableDB-based BlobDB
DECLARE_bool(use_blob_db);
DECLARE_uint64(blob_db_min_blob_size);
DECLARE_uint64(blob_db_bytes_per_sync);
DECLARE_bool(blob_db_enable_gc);
DECLARE_double(blob_db_gc_cutoff);
#endif // !ROCKSDB_LITE
+
+// Options for integrated BlobDB
+DECLARE_bool(allow_setting_blob_options_dynamically);
+DECLARE_bool(enable_blob_files);
+DECLARE_uint64(min_blob_size);
+DECLARE_uint64(blob_file_size);
+DECLARE_string(blob_compression_type);
+DECLARE_bool(enable_blob_garbage_collection);
+DECLARE_double(blob_garbage_collection_age_cutoff);
+DECLARE_double(blob_garbage_collection_force_threshold);
+DECLARE_uint64(blob_compaction_readahead_size);
+DECLARE_int32(blob_file_starting_level);
+DECLARE_bool(use_blob_cache);
+DECLARE_bool(use_shared_block_and_blob_cache);
+DECLARE_uint64(blob_cache_size);
+DECLARE_int32(blob_cache_numshardbits);
+DECLARE_int32(prepopulate_blob_cache);
+
DECLARE_int32(approximate_size_one_in);
DECLARE_bool(sync_fault_injection);
DECLARE_bool(skip_verifydb);
DECLARE_bool(enable_compaction_filter);
DECLARE_bool(paranoid_file_checks);
+DECLARE_bool(fail_if_options_file_error);
+DECLARE_uint64(batch_protection_bytes_per_key);
+DECLARE_uint32(memtable_protection_bytes_per_key);
-const long KB = 1024;
-const int kRandomValueMaxFactor = 3;
-const int kValueMaxLen = 100;
+DECLARE_uint64(user_timestamp_size);
+DECLARE_string(secondary_cache_uri);
+DECLARE_int32(secondary_cache_fault_one_in);
-// wrapped posix or hdfs environment
-extern ROCKSDB_NAMESPACE::DbStressEnvWrapper* db_stress_env;
-#ifndef NDEBUG
-namespace ROCKSDB_NAMESPACE {
-class FaultInjectionTestFS;
-} // namespace ROCKSDB_NAMESPACE
+DECLARE_int32(prepopulate_block_cache);
+
+DECLARE_bool(two_write_queues);
+#ifndef ROCKSDB_LITE
+DECLARE_bool(use_only_the_last_commit_time_batch_for_recovery);
+DECLARE_uint64(wp_snapshot_cache_bits);
+DECLARE_uint64(wp_commit_cache_bits);
+#endif // !ROCKSDB_LITE
+
+DECLARE_bool(adaptive_readahead);
+DECLARE_bool(async_io);
+DECLARE_string(wal_compression);
+DECLARE_bool(verify_sst_unique_id_in_manifest);
+
+DECLARE_int32(create_timestamped_snapshot_one_in);
+
+DECLARE_bool(allow_data_in_errors);
+
+// Tiered storage
+DECLARE_bool(enable_tiered_storage); // set last_level_temperature
+DECLARE_int64(preclude_last_level_data_seconds);
+DECLARE_int64(preserve_internal_time_seconds);
+
+DECLARE_int32(verify_iterator_with_expected_state_one_in);
+DECLARE_bool(preserve_unverified_changes);
+
+DECLARE_uint64(readahead_size);
+DECLARE_uint64(initial_auto_readahead_size);
+DECLARE_uint64(max_auto_readahead_size);
+DECLARE_uint64(num_file_reads_for_auto_readahead);
+
+constexpr long KB = 1024;
+constexpr int kRandomValueMaxFactor = 3;
+constexpr int kValueMaxLen = 100;
+
+// wrapped posix environment
+extern ROCKSDB_NAMESPACE::Env* db_stress_env;
+extern ROCKSDB_NAMESPACE::Env* db_stress_listener_env;
extern std::shared_ptr<ROCKSDB_NAMESPACE::FaultInjectionTestFS> fault_fs_guard;
-#endif
extern enum ROCKSDB_NAMESPACE::CompressionType compression_type_e;
extern enum ROCKSDB_NAMESPACE::CompressionType bottommost_compression_type_e;
#pragma warning(pop)
#endif
-// convert long to a big-endian slice key
-extern inline std::string GetStringFromInt(int64_t val) {
- std::string little_endian_key;
- std::string big_endian_key;
- PutFixed64(&little_endian_key, val);
- assert(little_endian_key.size() == sizeof(val));
- big_endian_key.resize(sizeof(val));
- for (size_t i = 0; i < sizeof(val); ++i) {
- big_endian_key[i] = little_endian_key[sizeof(val) - 1 - i];
+// Append `val` to `*key` in fixed-width big-endian format
+extern inline void AppendIntToString(uint64_t val, std::string* key) {
+ // PutFixed64 uses little endian
+ PutFixed64(key, val);
+ // Reverse to get big endian
+ char* int_data = &((*key)[key->size() - sizeof(uint64_t)]);
+ for (size_t i = 0; i < sizeof(uint64_t) / 2; ++i) {
+ std::swap(int_data[i], int_data[sizeof(uint64_t) - 1 - i]);
}
- return big_endian_key;
}
// A struct for maintaining the parameters for generating variable length keys
uint64_t window = key_gen_ctx.window;
size_t levels = key_gen_ctx.weights.size();
std::string key;
+ // Over-reserve and for now do not bother `shrink_to_fit()` since the key
+ // strings are transient.
+ key.reserve(FLAGS_max_key_len * 8);
+ uint64_t window_idx = static_cast<uint64_t>(val) / window;
+ uint64_t offset = static_cast<uint64_t>(val) % window;
for (size_t level = 0; level < levels; ++level) {
uint64_t weight = key_gen_ctx.weights[level];
- uint64_t offset = static_cast<uint64_t>(val) % window;
- uint64_t mult = static_cast<uint64_t>(val) / window;
- uint64_t pfx = mult * weight + (offset >= weight ? weight - 1 : offset);
- key.append(GetStringFromInt(pfx));
+ uint64_t pfx;
+ if (level == 0) {
+ pfx = window_idx * weight;
+ } else {
+ pfx = 0;
+ }
+ pfx += offset >= weight ? weight - 1 : offset;
+ AppendIntToString(pfx, &key);
if (offset < weight) {
// Use the bottom 3 bits of offset as the number of trailing 'x's in the
// key. If the next key is going to be of the next level, then skip the
- // trailer as it would break ordering. If the key length is already at max,
- // skip the trailer.
+ // trailer as it would break ordering. If the key length is already at
+ // max, skip the trailer.
if (offset < weight - 1 && level < levels - 1) {
size_t trailer_len = offset & 0x7;
key.append(trailer_len, 'x');
}
break;
}
- val = offset - weight;
- window -= weight;
+ offset -= weight;
}
return key;
extern void DbVerificationThread(void* v);
+extern void TimestampedSnapshotsThread(void* v);
+
extern void PrintKeyValue(int cf, uint64_t key, const char* value, size_t sz);
extern int64_t GenerateOneKey(ThreadState* thread, uint64_t iteration);
uint64_t iteration);
extern size_t GenerateValue(uint32_t rand, char* v, size_t max_sz);
+extern uint32_t GetValueBase(Slice s);
+
+extern WideColumns GenerateWideColumns(uint32_t value_base, const Slice& slice);
+extern WideColumns GenerateExpectedWideColumns(uint32_t value_base,
+ const Slice& slice);
extern StressTest* CreateCfConsistencyStressTest();
extern StressTest* CreateBatchedOpsStressTest();
extern StressTest* CreateNonBatchedOpsStressTest();
+extern StressTest* CreateMultiOpsTxnsStressTest();
+extern void CheckAndSetOptionsForMultiOpsTxnStressTest();
extern void InitializeHotKeyGenerator(double alpha);
extern int64_t GetOneHotKeyID(double rand_seed, int64_t max_key);
+extern std::string GetNowNanos();
+
std::shared_ptr<FileChecksumGenFactory> GetFileChecksumImpl(
const std::string& name);
+
+Status DeleteFilesInDirectory(const std::string& dirname);
+Status SaveFilesInDirectory(const std::string& src_dirname,
+ const std::string& dst_dirname);
+Status DestroyUnverifiedSubdir(const std::string& dirname);
+Status InitUnverifiedSubdir(const std::string& dirname);
} // namespace ROCKSDB_NAMESPACE
#endif // GFLAGS
projects / ceph.git / blobdiff