#include "rocksdb/table_properties.h"
+#include "db/seqno_to_time_mapping.h"
+#include "port/malloc.h"
#include "port/port.h"
#include "rocksdb/env.h"
-#include "rocksdb/iterator.h"
-#include "table/block_based/block.h"
-#include "table/internal_iterator.h"
+#include "rocksdb/unique_id.h"
#include "table/table_properties_internal.h"
+#include "table/unique_id_impl.h"
+#include "util/random.h"
#include "util/string_util.h"
namespace ROCKSDB_NAMESPACE {
const uint32_t TablePropertiesCollectorFactory::Context::kUnknownColumnFamily =
- port::kMaxInt32;
+ std::numeric_limits<int32_t>::max();
namespace {
- void AppendProperty(
- std::string& props,
- const std::string& key,
- const std::string& value,
- const std::string& prop_delim,
- const std::string& kv_delim) {
- props.append(key);
- props.append(kv_delim);
- props.append(value);
- props.append(prop_delim);
- }
-
- template <class TValue>
- void AppendProperty(
- std::string& props,
- const std::string& key,
- const TValue& value,
- const std::string& prop_delim,
- const std::string& kv_delim) {
- AppendProperty(
- props, key, ToString(value), prop_delim, kv_delim
- );
- }
+void AppendProperty(std::string& props, const std::string& key,
+ const std::string& value, const std::string& prop_delim,
+ const std::string& kv_delim) {
+ props.append(key);
+ props.append(kv_delim);
+ props.append(value);
+ props.append(prop_delim);
+}
- // Seek to the specified meta block.
- // Return true if it successfully seeks to that block.
- Status SeekToMetaBlock(InternalIterator* meta_iter,
- const std::string& block_name, bool* is_found,
- BlockHandle* block_handle = nullptr) {
- if (block_handle != nullptr) {
- *block_handle = BlockHandle::NullBlockHandle();
- }
- *is_found = true;
- meta_iter->Seek(block_name);
- if (meta_iter->status().ok()) {
- if (meta_iter->Valid() && meta_iter->key() == block_name) {
- *is_found = true;
- if (block_handle) {
- Slice v = meta_iter->value();
- return block_handle->DecodeFrom(&v);
- }
- } else {
- *is_found = false;
- return Status::OK();
- }
- }
- return meta_iter->status();
- }
+template <class TValue>
+void AppendProperty(std::string& props, const std::string& key,
+ const TValue& value, const std::string& prop_delim,
+ const std::string& kv_delim) {
+ AppendProperty(props, key, std::to_string(value), prop_delim, kv_delim);
}
+} // namespace
-std::string TableProperties::ToString(
- const std::string& prop_delim,
- const std::string& kv_delim) const {
+std::string TableProperties::ToString(const std::string& prop_delim,
+ const std::string& kv_delim) const {
std::string result;
result.reserve(1024);
if (index_partitions != 0) {
AppendProperty(result, "# index partitions", index_partitions, prop_delim,
kv_delim);
- AppendProperty(result, "top-level index size", top_level_index_size, prop_delim,
- kv_delim);
+ AppendProperty(result, "top-level index size", top_level_index_size,
+ prop_delim, kv_delim);
}
AppendProperty(result, "filter block size", filter_size, prop_delim,
kv_delim);
+ AppendProperty(result, "# entries for filter", num_filter_entries, prop_delim,
+ kv_delim);
AppendProperty(result, "(estimated) table size",
data_size + index_size + filter_size, prop_delim, kv_delim);
ROCKSDB_NAMESPACE::TablePropertiesCollectorFactory::
Context::kUnknownColumnFamily
? std::string("N/A")
- : ROCKSDB_NAMESPACE::ToString(column_family_id),
+ : std::to_string(column_family_id),
prop_delim, kv_delim);
AppendProperty(
result, "column family name",
AppendProperty(result, "file creation time", file_creation_time, prop_delim,
kv_delim);
+ AppendProperty(result, "slow compression estimated data size",
+ slow_compression_estimated_data_size, prop_delim, kv_delim);
+ AppendProperty(result, "fast compression estimated data size",
+ fast_compression_estimated_data_size, prop_delim, kv_delim);
+
// DB identity and DB session ID
AppendProperty(result, "DB identity", db_id, prop_delim, kv_delim);
AppendProperty(result, "DB session identity", db_session_id, prop_delim,
kv_delim);
+ AppendProperty(result, "DB host id", db_host_id, prop_delim, kv_delim);
+ AppendProperty(result, "original file number", orig_file_number, prop_delim,
+ kv_delim);
+
+ // Unique ID, when available
+ std::string id;
+ Status s = GetUniqueIdFromTableProperties(*this, &id);
+ AppendProperty(result, "unique ID",
+ s.ok() ? UniqueIdToHumanString(id) : "N/A", prop_delim,
+ kv_delim);
+
+ SeqnoToTimeMapping seq_time_mapping;
+ s = seq_time_mapping.Add(seqno_to_time_mapping);
+ AppendProperty(result, "Sequence number to time mapping",
+ s.ok() ? seq_time_mapping.ToHumanString() : "N/A", prop_delim,
+ kv_delim);
return result;
}
raw_value_size += tp.raw_value_size;
num_data_blocks += tp.num_data_blocks;
num_entries += tp.num_entries;
+ num_filter_entries += tp.num_filter_entries;
num_deletions += tp.num_deletions;
num_merge_operands += tp.num_merge_operands;
num_range_deletions += tp.num_range_deletions;
+ slow_compression_estimated_data_size +=
+ tp.slow_compression_estimated_data_size;
+ fast_compression_estimated_data_size +=
+ tp.fast_compression_estimated_data_size;
+}
+
+std::map<std::string, uint64_t>
+TableProperties::GetAggregatablePropertiesAsMap() const {
+ std::map<std::string, uint64_t> rv;
+ rv["data_size"] = data_size;
+ rv["index_size"] = index_size;
+ rv["index_partitions"] = index_partitions;
+ rv["top_level_index_size"] = top_level_index_size;
+ rv["filter_size"] = filter_size;
+ rv["raw_key_size"] = raw_key_size;
+ rv["raw_value_size"] = raw_value_size;
+ rv["num_data_blocks"] = num_data_blocks;
+ rv["num_entries"] = num_entries;
+ rv["num_filter_entries"] = num_filter_entries;
+ rv["num_deletions"] = num_deletions;
+ rv["num_merge_operands"] = num_merge_operands;
+ rv["num_range_deletions"] = num_range_deletions;
+ rv["slow_compression_estimated_data_size"] =
+ slow_compression_estimated_data_size;
+ rv["fast_compression_estimated_data_size"] =
+ fast_compression_estimated_data_size;
+ return rv;
+}
+
+// WARNING: manual update to this function is needed
+// whenever a new string property is added to TableProperties
+// to reduce approximation error.
+//
+// TODO: eliminate the need of manually updating this function
+// for new string properties
+std::size_t TableProperties::ApproximateMemoryUsage() const {
+ std::size_t usage = 0;
+#ifdef ROCKSDB_MALLOC_USABLE_SIZE
+ usage += malloc_usable_size((void*)this);
+#else
+ usage += sizeof(*this);
+#endif // ROCKSDB_MALLOC_USABLE_SIZE
+
+ std::size_t string_props_mem_usage =
+ db_id.size() + db_session_id.size() + db_host_id.size() +
+ column_family_name.size() + filter_policy_name.size() +
+ comparator_name.size() + merge_operator_name.size() +
+ prefix_extractor_name.size() + property_collectors_names.size() +
+ compression_name.size() + compression_options.size();
+ usage += string_props_mem_usage;
+
+ for (auto iter = user_collected_properties.begin();
+ iter != user_collected_properties.end(); ++iter) {
+ usage += (iter->first.size() + iter->second.size());
+ }
+
+ return usage;
}
const std::string TablePropertiesNames::kDbId = "rocksdb.creating.db.identity";
"rocksdb.creating.session.identity";
const std::string TablePropertiesNames::kDbHostId =
"rocksdb.creating.host.identity";
-const std::string TablePropertiesNames::kDataSize =
- "rocksdb.data.size";
-const std::string TablePropertiesNames::kIndexSize =
- "rocksdb.index.size";
+const std::string TablePropertiesNames::kOriginalFileNumber =
+ "rocksdb.original.file.number";
+const std::string TablePropertiesNames::kDataSize = "rocksdb.data.size";
+const std::string TablePropertiesNames::kIndexSize = "rocksdb.index.size";
const std::string TablePropertiesNames::kIndexPartitions =
"rocksdb.index.partitions";
const std::string TablePropertiesNames::kTopLevelIndexSize =
"rocksdb.index.key.is.user.key";
const std::string TablePropertiesNames::kIndexValueIsDeltaEncoded =
"rocksdb.index.value.is.delta.encoded";
-const std::string TablePropertiesNames::kFilterSize =
- "rocksdb.filter.size";
-const std::string TablePropertiesNames::kRawKeySize =
- "rocksdb.raw.key.size";
+const std::string TablePropertiesNames::kFilterSize = "rocksdb.filter.size";
+const std::string TablePropertiesNames::kRawKeySize = "rocksdb.raw.key.size";
const std::string TablePropertiesNames::kRawValueSize =
"rocksdb.raw.value.size";
const std::string TablePropertiesNames::kNumDataBlocks =
"rocksdb.num.data.blocks";
-const std::string TablePropertiesNames::kNumEntries =
- "rocksdb.num.entries";
+const std::string TablePropertiesNames::kNumEntries = "rocksdb.num.entries";
+const std::string TablePropertiesNames::kNumFilterEntries =
+ "rocksdb.num.filter_entries";
const std::string TablePropertiesNames::kDeletedKeys = "rocksdb.deleted.keys";
const std::string TablePropertiesNames::kMergeOperands =
"rocksdb.merge.operands";
const std::string TablePropertiesNames::kNumRangeDeletions =
"rocksdb.num.range-deletions";
-const std::string TablePropertiesNames::kFilterPolicy =
- "rocksdb.filter.policy";
+const std::string TablePropertiesNames::kFilterPolicy = "rocksdb.filter.policy";
const std::string TablePropertiesNames::kFormatVersion =
"rocksdb.format.version";
const std::string TablePropertiesNames::kFixedKeyLen =
"rocksdb.oldest.key.time";
const std::string TablePropertiesNames::kFileCreationTime =
"rocksdb.file.creation.time";
+const std::string TablePropertiesNames::kSlowCompressionEstimatedDataSize =
+ "rocksdb.sample_for_compression.slow.data.size";
+const std::string TablePropertiesNames::kFastCompressionEstimatedDataSize =
+ "rocksdb.sample_for_compression.fast.data.size";
+const std::string TablePropertiesNames::kSequenceNumberTimeMapping =
+ "rocksdb.seqno.time.map";
-extern const std::string kPropertiesBlock = "rocksdb.properties";
-// Old property block name for backward compatibility
-extern const std::string kPropertiesBlockOldName = "rocksdb.stats";
-extern const std::string kCompressionDictBlock = "rocksdb.compression_dict";
-extern const std::string kRangeDelBlock = "rocksdb.range_del";
-
-// Seek to the properties block.
-// Return true if it successfully seeks to the properties block.
-Status SeekToPropertiesBlock(InternalIterator* meta_iter, bool* is_found) {
- Status status = SeekToMetaBlock(meta_iter, kPropertiesBlock, is_found);
- if (!*is_found && status.ok()) {
- status = SeekToMetaBlock(meta_iter, kPropertiesBlockOldName, is_found);
- }
- return status;
-}
-
-// Seek to the compression dictionary block.
-// Return true if it successfully seeks to that block.
-Status SeekToCompressionDictBlock(InternalIterator* meta_iter, bool* is_found,
- BlockHandle* block_handle) {
- return SeekToMetaBlock(meta_iter, kCompressionDictBlock, is_found, block_handle);
-}
+#ifndef NDEBUG
+// WARNING: TEST_SetRandomTableProperties assumes the following layout of
+// TableProperties
+//
+// struct TableProperties {
+// int64_t orig_file_number = 0;
+// ...
+// ... int64_t properties only
+// ...
+// std::string db_id;
+// ...
+// ... std::string properties only
+// ...
+// std::string compression_options;
+// UserCollectedProperties user_collected_properties;
+// ...
+// ... Other extra properties: non-int64_t/non-std::string properties only
+// ...
+// }
+void TEST_SetRandomTableProperties(TableProperties* props) {
+ Random* r = Random::GetTLSInstance();
+ uint64_t* pu = &props->orig_file_number;
+ assert(static_cast<void*>(pu) == static_cast<void*>(props));
+ std::string* ps = &props->db_id;
+ const uint64_t* const pu_end = reinterpret_cast<const uint64_t*>(ps);
+ // Use the last string property's address instead of
+ // the first extra property (e.g `user_collected_properties`)'s address
+ // in the for-loop to avoid advancing pointer to pointing to
+ // potential non-zero padding bytes between these two addresses due to
+ // user_collected_properties's alignment requirement
+ const std::string* const ps_end_inclusive = &props->compression_options;
-Status SeekToRangeDelBlock(InternalIterator* meta_iter, bool* is_found,
- BlockHandle* block_handle = nullptr) {
- return SeekToMetaBlock(meta_iter, kRangeDelBlock, is_found, block_handle);
+ for (; pu < pu_end; ++pu) {
+ *pu = r->Next64();
+ }
+ assert(static_cast<void*>(pu) == static_cast<void*>(ps));
+ for (; ps <= ps_end_inclusive; ++ps) {
+ *ps = r->RandomBinaryString(13);
+ }
}
+#endif
} // namespace ROCKSDB_NAMESPACE
projects / ceph.git / blobdiff