1 // Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
2 // This source code is licensed under both the GPLv2 (found in the
3 // COPYING file in the root directory) and Apache 2.0 License
4 // (found in the LICENSE.Apache file in the root directory).
6 #include "options/cf_options.h"
13 #include "options/configurable_helper.h"
14 #include "options/db_options.h"
15 #include "options/options_helper.h"
16 #include "options/options_parser.h"
17 #include "port/port.h"
18 #include "rocksdb/concurrent_task_limiter.h"
19 #include "rocksdb/configurable.h"
20 #include "rocksdb/convenience.h"
21 #include "rocksdb/env.h"
22 #include "rocksdb/file_system.h"
23 #include "rocksdb/merge_operator.h"
24 #include "rocksdb/options.h"
25 #include "rocksdb/table.h"
26 #include "rocksdb/utilities/object_registry.h"
27 #include "rocksdb/utilities/options_type.h"
28 #include "util/cast_util.h"
30 namespace ROCKSDB_NAMESPACE
{
31 // offset_of is used to get the offset of a class data member
32 // ex: offset_of(&ColumnFamilyOptions::num_levels)
33 // This call will return the offset of num_levels in ColumnFamilyOptions class
35 // This is the same as offsetof() but allow us to work with non standard-layout
36 // classes and structures
38 // http://en.cppreference.com/w/cpp/concept/StandardLayoutType
39 // https://gist.github.com/graphitemaster/494f21190bb2c63c5516
41 static ColumnFamilyOptions dummy_cf_options
;
42 template <typename T1
>
43 int offset_of(T1
ColumnFamilyOptions::*member
) {
44 return int(size_t(&(dummy_cf_options
.*member
)) - size_t(&dummy_cf_options
));
46 template <typename T1
>
47 int offset_of(T1
AdvancedColumnFamilyOptions::*member
) {
48 return int(size_t(&(dummy_cf_options
.*member
)) - size_t(&dummy_cf_options
));
51 static Status
ParseCompressionOptions(const std::string
& value
,
52 const std::string
& name
,
53 CompressionOptions
& compression_opts
) {
55 size_t end
= value
.find(':');
56 if (end
== std::string::npos
) {
57 return Status::InvalidArgument("unable to parse the specified CF option " +
60 compression_opts
.window_bits
= ParseInt(value
.substr(start
, end
- start
));
62 end
= value
.find(':', start
);
63 if (end
== std::string::npos
) {
64 return Status::InvalidArgument("unable to parse the specified CF option " +
67 compression_opts
.level
= ParseInt(value
.substr(start
, end
- start
));
69 if (start
>= value
.size()) {
70 return Status::InvalidArgument("unable to parse the specified CF option " +
73 end
= value
.find(':', start
);
74 compression_opts
.strategy
=
75 ParseInt(value
.substr(start
, value
.size() - start
));
76 // max_dict_bytes is optional for backwards compatibility
77 if (end
!= std::string::npos
) {
79 if (start
>= value
.size()) {
80 return Status::InvalidArgument(
81 "unable to parse the specified CF option " + name
);
83 compression_opts
.max_dict_bytes
=
84 ParseInt(value
.substr(start
, value
.size() - start
));
85 end
= value
.find(':', start
);
87 // zstd_max_train_bytes is optional for backwards compatibility
88 if (end
!= std::string::npos
) {
90 if (start
>= value
.size()) {
91 return Status::InvalidArgument(
92 "unable to parse the specified CF option " + name
);
94 compression_opts
.zstd_max_train_bytes
=
95 ParseInt(value
.substr(start
, value
.size() - start
));
96 end
= value
.find(':', start
);
99 // parallel_threads is optional for backwards compatibility
100 if (end
!= std::string::npos
) {
102 if (start
>= value
.size()) {
103 return Status::InvalidArgument(
104 "unable to parse the specified CF option " + name
);
106 // Since parallel_threads comes before enabled but was added optionally
107 // later, we need to check if this is the final token (meaning it is the
108 // enabled bit), or if there is another token (meaning this one is
110 end
= value
.find(':', start
);
111 if (end
!= std::string::npos
) {
112 compression_opts
.parallel_threads
=
113 ParseInt(value
.substr(start
, value
.size() - start
));
115 // parallel_threads is not serialized with this format, but enabled is
116 compression_opts
.parallel_threads
= CompressionOptions().parallel_threads
;
117 compression_opts
.enabled
=
118 ParseBoolean("", value
.substr(start
, value
.size() - start
));
122 // enabled is optional for backwards compatibility
123 if (end
!= std::string::npos
) {
125 if (start
>= value
.size()) {
126 return Status::InvalidArgument(
127 "unable to parse the specified CF option " + name
);
129 compression_opts
.enabled
=
130 ParseBoolean("", value
.substr(start
, value
.size() - start
));
135 const std::string kOptNameBMCompOpts
= "bottommost_compression_opts";
136 const std::string kOptNameCompOpts
= "compression_opts";
138 // OptionTypeInfo map for CompressionOptions
139 static std::unordered_map
<std::string
, OptionTypeInfo
>
140 compression_options_type_info
= {
142 {offsetof(struct CompressionOptions
, window_bits
), OptionType::kInt
,
143 OptionVerificationType::kNormal
, OptionTypeFlags::kMutable
}},
145 {offsetof(struct CompressionOptions
, level
), OptionType::kInt
,
146 OptionVerificationType::kNormal
, OptionTypeFlags::kMutable
}},
148 {offsetof(struct CompressionOptions
, strategy
), OptionType::kInt
,
149 OptionVerificationType::kNormal
, OptionTypeFlags::kMutable
}},
151 {offsetof(struct CompressionOptions
, max_dict_bytes
), OptionType::kInt
,
152 OptionVerificationType::kNormal
, OptionTypeFlags::kMutable
}},
153 {"zstd_max_train_bytes",
154 {offsetof(struct CompressionOptions
, zstd_max_train_bytes
),
155 OptionType::kUInt32T
, OptionVerificationType::kNormal
,
156 OptionTypeFlags::kMutable
}},
158 {offsetof(struct CompressionOptions
, parallel_threads
),
159 OptionType::kUInt32T
, OptionVerificationType::kNormal
,
160 OptionTypeFlags::kMutable
}},
162 {offsetof(struct CompressionOptions
, enabled
), OptionType::kBoolean
,
163 OptionVerificationType::kNormal
, OptionTypeFlags::kMutable
}},
166 static std::unordered_map
<std::string
, OptionTypeInfo
>
167 fifo_compaction_options_type_info
= {
168 {"max_table_files_size",
169 {offsetof(struct CompactionOptionsFIFO
, max_table_files_size
),
170 OptionType::kUInt64T
, OptionVerificationType::kNormal
,
171 OptionTypeFlags::kMutable
}},
173 {0, OptionType::kUInt64T
, OptionVerificationType::kDeprecated
,
174 OptionTypeFlags::kNone
}},
176 {offsetof(struct CompactionOptionsFIFO
, allow_compaction
),
177 OptionType::kBoolean
, OptionVerificationType::kNormal
,
178 OptionTypeFlags::kMutable
}},
181 static std::unordered_map
<std::string
, OptionTypeInfo
>
182 universal_compaction_options_type_info
= {
184 {offsetof(class CompactionOptionsUniversal
, size_ratio
),
185 OptionType::kUInt
, OptionVerificationType::kNormal
,
186 OptionTypeFlags::kMutable
}},
188 {offsetof(class CompactionOptionsUniversal
, min_merge_width
),
189 OptionType::kUInt
, OptionVerificationType::kNormal
,
190 OptionTypeFlags::kMutable
}},
192 {offsetof(class CompactionOptionsUniversal
, max_merge_width
),
193 OptionType::kUInt
, OptionVerificationType::kNormal
,
194 OptionTypeFlags::kMutable
}},
195 {"max_size_amplification_percent",
196 {offsetof(class CompactionOptionsUniversal
,
197 max_size_amplification_percent
),
198 OptionType::kUInt
, OptionVerificationType::kNormal
,
199 OptionTypeFlags::kMutable
}},
200 {"compression_size_percent",
201 {offsetof(class CompactionOptionsUniversal
, compression_size_percent
),
202 OptionType::kInt
, OptionVerificationType::kNormal
,
203 OptionTypeFlags::kMutable
}},
205 {offsetof(class CompactionOptionsUniversal
, stop_style
),
206 OptionType::kCompactionStopStyle
, OptionVerificationType::kNormal
,
207 OptionTypeFlags::kMutable
}},
208 {"allow_trivial_move",
209 {offsetof(class CompactionOptionsUniversal
, allow_trivial_move
),
210 OptionType::kBoolean
, OptionVerificationType::kNormal
,
211 OptionTypeFlags::kMutable
}}};
213 static std::unordered_map
<std::string
, OptionTypeInfo
>
214 cf_mutable_options_type_info
= {
215 {"report_bg_io_stats",
216 {offsetof(struct MutableCFOptions
, report_bg_io_stats
),
217 OptionType::kBoolean
, OptionVerificationType::kNormal
,
218 OptionTypeFlags::kMutable
}},
219 {"disable_auto_compactions",
220 {offsetof(struct MutableCFOptions
, disable_auto_compactions
),
221 OptionType::kBoolean
, OptionVerificationType::kNormal
,
222 OptionTypeFlags::kMutable
}},
224 {0, OptionType::kBoolean
, OptionVerificationType::kDeprecated
,
225 OptionTypeFlags::kMutable
}},
226 {"check_flush_compaction_key_order",
227 {offsetof(struct MutableCFOptions
, check_flush_compaction_key_order
),
228 OptionType::kBoolean
, OptionVerificationType::kNormal
,
229 OptionTypeFlags::kMutable
}},
230 {"paranoid_file_checks",
231 {offsetof(struct MutableCFOptions
, paranoid_file_checks
),
232 OptionType::kBoolean
, OptionVerificationType::kNormal
,
233 OptionTypeFlags::kMutable
}},
234 {"verify_checksums_in_compaction",
235 {0, OptionType::kBoolean
, OptionVerificationType::kDeprecated
,
236 OptionTypeFlags::kMutable
}},
237 {"soft_pending_compaction_bytes_limit",
238 {offsetof(struct MutableCFOptions
,
239 soft_pending_compaction_bytes_limit
),
240 OptionType::kUInt64T
, OptionVerificationType::kNormal
,
241 OptionTypeFlags::kMutable
}},
242 {"hard_pending_compaction_bytes_limit",
243 {offsetof(struct MutableCFOptions
,
244 hard_pending_compaction_bytes_limit
),
245 OptionType::kUInt64T
, OptionVerificationType::kNormal
,
246 OptionTypeFlags::kMutable
}},
248 {0, OptionType::kDouble
, OptionVerificationType::kDeprecated
,
249 OptionTypeFlags::kMutable
}},
251 {0, OptionType::kDouble
, OptionVerificationType::kDeprecated
,
252 OptionTypeFlags::kMutable
}},
253 {"max_compaction_bytes",
254 {offsetof(struct MutableCFOptions
, max_compaction_bytes
),
255 OptionType::kUInt64T
, OptionVerificationType::kNormal
,
256 OptionTypeFlags::kMutable
}},
257 {"expanded_compaction_factor",
258 {0, OptionType::kInt
, OptionVerificationType::kDeprecated
,
259 OptionTypeFlags::kMutable
}},
260 {"level0_file_num_compaction_trigger",
261 {offsetof(struct MutableCFOptions
, level0_file_num_compaction_trigger
),
262 OptionType::kInt
, OptionVerificationType::kNormal
,
263 OptionTypeFlags::kMutable
}},
264 {"level0_slowdown_writes_trigger",
265 {offsetof(struct MutableCFOptions
, level0_slowdown_writes_trigger
),
266 OptionType::kInt
, OptionVerificationType::kNormal
,
267 OptionTypeFlags::kMutable
}},
268 {"level0_stop_writes_trigger",
269 {offsetof(struct MutableCFOptions
, level0_stop_writes_trigger
),
270 OptionType::kInt
, OptionVerificationType::kNormal
,
271 OptionTypeFlags::kMutable
}},
272 {"max_grandparent_overlap_factor",
273 {0, OptionType::kInt
, OptionVerificationType::kDeprecated
,
274 OptionTypeFlags::kMutable
}},
275 {"max_write_buffer_number",
276 {offsetof(struct MutableCFOptions
, max_write_buffer_number
),
277 OptionType::kInt
, OptionVerificationType::kNormal
,
278 OptionTypeFlags::kMutable
}},
279 {"source_compaction_factor",
280 {0, OptionType::kInt
, OptionVerificationType::kDeprecated
,
281 OptionTypeFlags::kMutable
}},
282 {"target_file_size_multiplier",
283 {offsetof(struct MutableCFOptions
, target_file_size_multiplier
),
284 OptionType::kInt
, OptionVerificationType::kNormal
,
285 OptionTypeFlags::kMutable
}},
287 {offsetof(struct MutableCFOptions
, arena_block_size
),
288 OptionType::kSizeT
, OptionVerificationType::kNormal
,
289 OptionTypeFlags::kMutable
}},
290 {"inplace_update_num_locks",
291 {offsetof(struct MutableCFOptions
, inplace_update_num_locks
),
292 OptionType::kSizeT
, OptionVerificationType::kNormal
,
293 OptionTypeFlags::kMutable
}},
294 {"max_successive_merges",
295 {offsetof(struct MutableCFOptions
, max_successive_merges
),
296 OptionType::kSizeT
, OptionVerificationType::kNormal
,
297 OptionTypeFlags::kMutable
}},
298 {"memtable_huge_page_size",
299 {offsetof(struct MutableCFOptions
, memtable_huge_page_size
),
300 OptionType::kSizeT
, OptionVerificationType::kNormal
,
301 OptionTypeFlags::kMutable
}},
302 {"memtable_prefix_bloom_huge_page_tlb_size",
303 {0, OptionType::kSizeT
, OptionVerificationType::kDeprecated
,
304 OptionTypeFlags::kMutable
}},
305 {"write_buffer_size",
306 {offsetof(struct MutableCFOptions
, write_buffer_size
),
307 OptionType::kSizeT
, OptionVerificationType::kNormal
,
308 OptionTypeFlags::kMutable
}},
309 {"memtable_prefix_bloom_bits",
310 {0, OptionType::kUInt32T
, OptionVerificationType::kDeprecated
,
311 OptionTypeFlags::kMutable
}},
312 {"memtable_prefix_bloom_size_ratio",
313 {offsetof(struct MutableCFOptions
, memtable_prefix_bloom_size_ratio
),
314 OptionType::kDouble
, OptionVerificationType::kNormal
,
315 OptionTypeFlags::kMutable
}},
316 {"memtable_prefix_bloom_probes",
317 {0, OptionType::kUInt32T
, OptionVerificationType::kDeprecated
,
318 OptionTypeFlags::kMutable
}},
319 {"memtable_whole_key_filtering",
320 {offsetof(struct MutableCFOptions
, memtable_whole_key_filtering
),
321 OptionType::kBoolean
, OptionVerificationType::kNormal
,
322 OptionTypeFlags::kMutable
}},
323 {"min_partial_merge_operands",
324 {0, OptionType::kUInt32T
, OptionVerificationType::kDeprecated
,
325 OptionTypeFlags::kMutable
}},
326 {"max_bytes_for_level_base",
327 {offsetof(struct MutableCFOptions
, max_bytes_for_level_base
),
328 OptionType::kUInt64T
, OptionVerificationType::kNormal
,
329 OptionTypeFlags::kMutable
}},
330 {"snap_refresh_nanos",
331 {0, OptionType::kUInt64T
, OptionVerificationType::kDeprecated
,
332 OptionTypeFlags::kMutable
}},
333 {"max_bytes_for_level_multiplier",
334 {offsetof(struct MutableCFOptions
, max_bytes_for_level_multiplier
),
335 OptionType::kDouble
, OptionVerificationType::kNormal
,
336 OptionTypeFlags::kMutable
}},
337 {"max_bytes_for_level_multiplier_additional",
338 OptionTypeInfo::Vector
<int>(
339 offsetof(struct MutableCFOptions
,
340 max_bytes_for_level_multiplier_additional
),
341 OptionVerificationType::kNormal
, OptionTypeFlags::kMutable
,
342 {0, OptionType::kInt
})},
343 {"max_sequential_skip_in_iterations",
344 {offsetof(struct MutableCFOptions
, max_sequential_skip_in_iterations
),
345 OptionType::kUInt64T
, OptionVerificationType::kNormal
,
346 OptionTypeFlags::kMutable
}},
347 {"target_file_size_base",
348 {offsetof(struct MutableCFOptions
, target_file_size_base
),
349 OptionType::kUInt64T
, OptionVerificationType::kNormal
,
350 OptionTypeFlags::kMutable
}},
352 {offsetof(struct MutableCFOptions
, compression
),
353 OptionType::kCompressionType
, OptionVerificationType::kNormal
,
354 OptionTypeFlags::kMutable
}},
356 {offsetof(struct MutableCFOptions
, prefix_extractor
),
357 OptionType::kSliceTransform
, OptionVerificationType::kByNameAllowNull
,
358 OptionTypeFlags::kMutable
}},
359 {"compaction_options_fifo",
360 OptionTypeInfo::Struct(
361 "compaction_options_fifo", &fifo_compaction_options_type_info
,
362 offsetof(struct MutableCFOptions
, compaction_options_fifo
),
363 OptionVerificationType::kNormal
, OptionTypeFlags::kMutable
,
364 [](const ConfigOptions
& opts
, const std::string
& name
,
365 const std::string
& value
, char* addr
) {
366 // This is to handle backward compatibility, where
367 // compaction_options_fifo could be assigned a single scalar
368 // value, say, like "23", which would be assigned to
369 // max_table_files_size.
370 if (name
== "compaction_options_fifo" &&
371 value
.find("=") == std::string::npos
) {
372 // Old format. Parse just a single uint64_t value.
373 auto options
= reinterpret_cast<CompactionOptionsFIFO
*>(addr
);
374 options
->max_table_files_size
= ParseUint64(value
);
377 return OptionTypeInfo::ParseStruct(
378 opts
, "compaction_options_fifo",
379 &fifo_compaction_options_type_info
, name
, value
, addr
);
382 {"compaction_options_universal",
383 OptionTypeInfo::Struct(
384 "compaction_options_universal",
385 &universal_compaction_options_type_info
,
386 offsetof(struct MutableCFOptions
, compaction_options_universal
),
387 OptionVerificationType::kNormal
, OptionTypeFlags::kMutable
)},
389 {offsetof(struct MutableCFOptions
, ttl
), OptionType::kUInt64T
,
390 OptionVerificationType::kNormal
, OptionTypeFlags::kMutable
}},
391 {"periodic_compaction_seconds",
392 {offsetof(struct MutableCFOptions
, periodic_compaction_seconds
),
393 OptionType::kUInt64T
, OptionVerificationType::kNormal
,
394 OptionTypeFlags::kMutable
}},
395 {"enable_blob_files",
396 {offsetof(struct MutableCFOptions
, enable_blob_files
),
397 OptionType::kBoolean
, OptionVerificationType::kNormal
,
398 OptionTypeFlags::kMutable
}},
400 {offsetof(struct MutableCFOptions
, min_blob_size
),
401 OptionType::kUInt64T
, OptionVerificationType::kNormal
,
402 OptionTypeFlags::kMutable
}},
404 {offsetof(struct MutableCFOptions
, blob_file_size
),
405 OptionType::kUInt64T
, OptionVerificationType::kNormal
,
406 OptionTypeFlags::kMutable
}},
407 {"blob_compression_type",
408 {offsetof(struct MutableCFOptions
, blob_compression_type
),
409 OptionType::kCompressionType
, OptionVerificationType::kNormal
,
410 OptionTypeFlags::kMutable
}},
411 {"enable_blob_garbage_collection",
412 {offsetof(struct MutableCFOptions
, enable_blob_garbage_collection
),
413 OptionType::kBoolean
, OptionVerificationType::kNormal
,
414 OptionTypeFlags::kMutable
}},
415 {"blob_garbage_collection_age_cutoff",
416 {offsetof(struct MutableCFOptions
, blob_garbage_collection_age_cutoff
),
417 OptionType::kDouble
, OptionVerificationType::kNormal
,
418 OptionTypeFlags::kMutable
}},
419 {"sample_for_compression",
420 {offsetof(struct MutableCFOptions
, sample_for_compression
),
421 OptionType::kUInt64T
, OptionVerificationType::kNormal
,
422 OptionTypeFlags::kMutable
}},
423 {"bottommost_compression",
424 {offsetof(struct MutableCFOptions
, bottommost_compression
),
425 OptionType::kCompressionType
, OptionVerificationType::kNormal
,
426 OptionTypeFlags::kMutable
}},
428 OptionTypeInfo::Struct(
429 kOptNameCompOpts
, &compression_options_type_info
,
430 offsetof(struct MutableCFOptions
, compression_opts
),
431 OptionVerificationType::kNormal
,
432 (OptionTypeFlags::kMutable
| OptionTypeFlags::kCompareNever
),
433 [](const ConfigOptions
& opts
, const std::string
& name
,
434 const std::string
& value
, char* addr
) {
435 // This is to handle backward compatibility, where
436 // compression_options was a ":" separated list.
437 if (name
== kOptNameCompOpts
&&
438 value
.find("=") == std::string::npos
) {
440 reinterpret_cast<CompressionOptions
*>(addr
);
441 return ParseCompressionOptions(value
, name
, *compression
);
443 return OptionTypeInfo::ParseStruct(
444 opts
, kOptNameCompOpts
, &compression_options_type_info
,
449 OptionTypeInfo::Struct(
450 kOptNameBMCompOpts
, &compression_options_type_info
,
451 offsetof(struct MutableCFOptions
, bottommost_compression_opts
),
452 OptionVerificationType::kNormal
,
453 (OptionTypeFlags::kMutable
| OptionTypeFlags::kCompareNever
),
454 [](const ConfigOptions
& opts
, const std::string
& name
,
455 const std::string
& value
, char* addr
) {
456 // This is to handle backward compatibility, where
457 // compression_options was a ":" separated list.
458 if (name
== kOptNameBMCompOpts
&&
459 value
.find("=") == std::string::npos
) {
461 reinterpret_cast<CompressionOptions
*>(addr
);
462 return ParseCompressionOptions(value
, name
, *compression
);
464 return OptionTypeInfo::ParseStruct(
465 opts
, kOptNameBMCompOpts
, &compression_options_type_info
,
469 // End special case properties
472 static std::unordered_map
<std::string
, OptionTypeInfo
>
473 cf_immutable_options_type_info
= {
475 CompressionOptions compression_opts;
476 TablePropertiesCollectorFactories table_properties_collector_factories;
477 typedef std::vector<std::shared_ptr<TablePropertiesCollectorFactory>>
478 TablePropertiesCollectorFactories;
479 UpdateStatus (*inplace_callback)(char* existing_value,
480 uint34_t* existing_value_size,
482 std::string* merged_value);
483 std::vector<DbPath> cf_paths;
485 {"compaction_measure_io_stats",
486 {0, OptionType::kBoolean
, OptionVerificationType::kDeprecated
,
487 OptionTypeFlags::kNone
}},
488 {"inplace_update_support",
489 {offset_of(&ColumnFamilyOptions::inplace_update_support
),
490 OptionType::kBoolean
, OptionVerificationType::kNormal
,
491 OptionTypeFlags::kNone
}},
492 {"level_compaction_dynamic_level_bytes",
493 {offset_of(&ColumnFamilyOptions::level_compaction_dynamic_level_bytes
),
494 OptionType::kBoolean
, OptionVerificationType::kNormal
,
495 OptionTypeFlags::kNone
}},
496 {"optimize_filters_for_hits",
497 {offset_of(&ColumnFamilyOptions::optimize_filters_for_hits
),
498 OptionType::kBoolean
, OptionVerificationType::kNormal
,
499 OptionTypeFlags::kNone
}},
500 {"force_consistency_checks",
501 {offset_of(&ColumnFamilyOptions::force_consistency_checks
),
502 OptionType::kBoolean
, OptionVerificationType::kNormal
,
503 OptionTypeFlags::kNone
}},
504 {"purge_redundant_kvs_while_flush",
505 {offset_of(&ColumnFamilyOptions::purge_redundant_kvs_while_flush
),
506 OptionType::kBoolean
, OptionVerificationType::kDeprecated
,
507 OptionTypeFlags::kNone
}},
508 {"max_mem_compaction_level",
509 {0, OptionType::kInt
, OptionVerificationType::kDeprecated
,
510 OptionTypeFlags::kNone
}},
511 {"max_write_buffer_number_to_maintain",
512 {offset_of(&ColumnFamilyOptions::max_write_buffer_number_to_maintain
),
513 OptionType::kInt
, OptionVerificationType::kNormal
,
514 OptionTypeFlags::kNone
, 0}},
515 {"max_write_buffer_size_to_maintain",
516 {offset_of(&ColumnFamilyOptions::max_write_buffer_size_to_maintain
),
517 OptionType::kInt64T
, OptionVerificationType::kNormal
,
518 OptionTypeFlags::kNone
}},
519 {"min_write_buffer_number_to_merge",
520 {offset_of(&ColumnFamilyOptions::min_write_buffer_number_to_merge
),
521 OptionType::kInt
, OptionVerificationType::kNormal
,
522 OptionTypeFlags::kNone
, 0}},
524 {offset_of(&ColumnFamilyOptions::num_levels
), OptionType::kInt
,
525 OptionVerificationType::kNormal
, OptionTypeFlags::kNone
}},
527 {offset_of(&ColumnFamilyOptions::bloom_locality
), OptionType::kUInt32T
,
528 OptionVerificationType::kNormal
, OptionTypeFlags::kNone
}},
529 {"rate_limit_delay_max_milliseconds",
530 {0, OptionType::kUInt
, OptionVerificationType::kDeprecated
,
531 OptionTypeFlags::kNone
}},
532 {"compression_per_level",
533 OptionTypeInfo::Vector
<CompressionType
>(
534 offset_of(&ColumnFamilyOptions::compression_per_level
),
535 OptionVerificationType::kNormal
, OptionTypeFlags::kNone
,
536 {0, OptionType::kCompressionType
})},
538 {offset_of(&ColumnFamilyOptions::comparator
), OptionType::kComparator
,
539 OptionVerificationType::kByName
, OptionTypeFlags::kCompareLoose
,
540 // Parses the string and sets the corresponding comparator
541 [](const ConfigOptions
& /*opts*/, const std::string
& /*name*/,
542 const std::string
& value
, char* addr
) {
543 auto old_comparator
= reinterpret_cast<const Comparator
**>(addr
);
544 const Comparator
* new_comparator
= *old_comparator
;
545 Status status
= ObjectRegistry::NewInstance()->NewStaticObject(
546 value
, &new_comparator
);
548 *old_comparator
= new_comparator
;
553 {"memtable_insert_with_hint_prefix_extractor",
555 &ColumnFamilyOptions::memtable_insert_with_hint_prefix_extractor
),
556 OptionType::kSliceTransform
, OptionVerificationType::kByNameAllowNull
,
557 OptionTypeFlags::kNone
}},
559 {offset_of(&ColumnFamilyOptions::memtable_factory
),
560 OptionType::kMemTableRepFactory
, OptionVerificationType::kByName
,
561 OptionTypeFlags::kNone
}},
563 {offset_of(&ColumnFamilyOptions::memtable_factory
),
564 OptionType::kMemTableRepFactory
, OptionVerificationType::kAlias
,
565 OptionTypeFlags::kNone
,
566 // Parses the value string and updates the memtable_factory
567 [](const ConfigOptions
& /*opts*/, const std::string
& /*name*/,
568 const std::string
& value
, char* addr
) {
569 std::unique_ptr
<MemTableRepFactory
> new_mem_factory
;
570 Status s
= GetMemTableRepFactoryFromString(value
, &new_mem_factory
);
572 auto memtable_factory
=
573 reinterpret_cast<std::shared_ptr
<MemTableRepFactory
>*>(addr
);
574 memtable_factory
->reset(new_mem_factory
.release());
578 {"table_factory", OptionTypeInfo::AsCustomSharedPtr
<TableFactory
>(
579 offset_of(&ColumnFamilyOptions::table_factory
),
580 OptionVerificationType::kByName
,
581 (OptionTypeFlags::kCompareLoose
|
582 OptionTypeFlags::kStringNameOnly
|
583 OptionTypeFlags::kDontPrepare
))},
584 {"block_based_table_factory",
585 {offset_of(&ColumnFamilyOptions::table_factory
),
586 OptionType::kCustomizable
, OptionVerificationType::kAlias
,
587 OptionTypeFlags::kShared
| OptionTypeFlags::kCompareLoose
,
588 // Parses the input value and creates a BlockBasedTableFactory
589 [](const ConfigOptions
& opts
, const std::string
& name
,
590 const std::string
& value
, char* addr
) {
591 BlockBasedTableOptions
* old_opts
= nullptr;
593 reinterpret_cast<std::shared_ptr
<TableFactory
>*>(addr
);
594 if (table_factory
->get() != nullptr) {
596 table_factory
->get()->GetOptions
<BlockBasedTableOptions
>();
598 if (name
== "block_based_table_factory") {
599 std::unique_ptr
<TableFactory
> new_factory
;
600 if (old_opts
!= nullptr) {
601 new_factory
.reset(NewBlockBasedTableFactory(*old_opts
));
603 new_factory
.reset(NewBlockBasedTableFactory());
605 Status s
= new_factory
->ConfigureFromString(opts
, value
);
607 table_factory
->reset(new_factory
.release());
610 } else if (old_opts
!= nullptr) {
611 return table_factory
->get()->ConfigureOption(opts
, name
, value
);
613 return Status::NotFound("Mismatched table option: ", name
);
616 {"plain_table_factory",
617 {offset_of(&ColumnFamilyOptions::table_factory
),
618 OptionType::kCustomizable
, OptionVerificationType::kAlias
,
619 OptionTypeFlags::kShared
| OptionTypeFlags::kCompareLoose
,
620 // Parses the input value and creates a PlainTableFactory
621 [](const ConfigOptions
& opts
, const std::string
& name
,
622 const std::string
& value
, char* addr
) {
623 PlainTableOptions
* old_opts
= nullptr;
625 reinterpret_cast<std::shared_ptr
<TableFactory
>*>(addr
);
626 if (table_factory
->get() != nullptr) {
627 old_opts
= table_factory
->get()->GetOptions
<PlainTableOptions
>();
629 if (name
== "plain_table_factory") {
630 std::unique_ptr
<TableFactory
> new_factory
;
631 if (old_opts
!= nullptr) {
632 new_factory
.reset(NewPlainTableFactory(*old_opts
));
634 new_factory
.reset(NewPlainTableFactory());
636 Status s
= new_factory
->ConfigureFromString(opts
, value
);
638 table_factory
->reset(new_factory
.release());
641 } else if (old_opts
!= nullptr) {
642 return table_factory
->get()->ConfigureOption(opts
, name
, value
);
644 return Status::NotFound("Mismatched table option: ", name
);
647 {"compaction_filter",
648 {offset_of(&ColumnFamilyOptions::compaction_filter
),
649 OptionType::kCompactionFilter
, OptionVerificationType::kByName
,
650 OptionTypeFlags::kNone
}},
651 {"compaction_filter_factory",
652 {offset_of(&ColumnFamilyOptions::compaction_filter_factory
),
653 OptionType::kCompactionFilterFactory
, OptionVerificationType::kByName
,
654 OptionTypeFlags::kNone
}},
656 {offset_of(&ColumnFamilyOptions::merge_operator
),
657 OptionType::kMergeOperator
,
658 OptionVerificationType::kByNameAllowFromNull
,
659 OptionTypeFlags::kCompareLoose
,
660 // Parses the input value as a MergeOperator, updating the value
661 [](const ConfigOptions
& /*opts*/, const std::string
& /*name*/,
662 const std::string
& value
, char* addr
) {
663 auto mop
= reinterpret_cast<std::shared_ptr
<MergeOperator
>*>(addr
);
665 ObjectRegistry::NewInstance()->NewSharedObject
<MergeOperator
>(
667 // Only support static comparator for now.
674 {offset_of(&ColumnFamilyOptions::compaction_style
),
675 OptionType::kCompactionStyle
, OptionVerificationType::kNormal
,
676 OptionTypeFlags::kNone
}},
678 {offset_of(&ColumnFamilyOptions::compaction_pri
),
679 OptionType::kCompactionPri
, OptionVerificationType::kNormal
,
680 OptionTypeFlags::kNone
}},
683 const std::string
OptionsHelper::kCFOptionsName
= "ColumnFamilyOptions";
685 class ConfigurableMutableCFOptions
: public Configurable
{
687 ConfigurableMutableCFOptions(const MutableCFOptions
& mcf
) {
689 ConfigurableHelper::RegisterOptions(*this, &mutable_
,
690 &cf_mutable_options_type_info
);
694 MutableCFOptions mutable_
;
697 class ConfigurableCFOptions
: public ConfigurableMutableCFOptions
{
699 ConfigurableCFOptions(const ColumnFamilyOptions
& opts
,
700 const std::unordered_map
<std::string
, std::string
>* map
)
701 : ConfigurableMutableCFOptions(MutableCFOptions(opts
)),
705 ConfigurableHelper::RegisterOptions(*this, OptionsHelper::kCFOptionsName
,
707 &cf_immutable_options_type_info
);
711 Status
ConfigureOptions(
712 const ConfigOptions
& config_options
,
713 const std::unordered_map
<std::string
, std::string
>& opts_map
,
714 std::unordered_map
<std::string
, std::string
>* unused
) override
{
715 Status s
= ConfigurableHelper::ConfigureOptions(config_options
, *this,
718 cf_options_
= BuildColumnFamilyOptions(immutable_
, mutable_
);
719 s
= PrepareOptions(config_options
);
724 virtual const void* GetOptionsPtr(const std::string
& name
) const override
{
725 if (name
== OptionsHelper::kCFOptionsName
) {
728 return ConfigurableMutableCFOptions::GetOptionsPtr(name
);
732 bool OptionsAreEqual(const ConfigOptions
& config_options
,
733 const OptionTypeInfo
& opt_info
,
734 const std::string
& opt_name
, const void* const this_ptr
,
735 const void* const that_ptr
,
736 std::string
* mismatch
) const override
{
737 bool equals
= opt_info
.AreEqual(config_options
, opt_name
, this_ptr
,
739 if (!equals
&& opt_info
.IsByName()) {
740 if (opt_map_
== nullptr) {
743 const auto& iter
= opt_map_
->find(opt_name
);
744 if (iter
== opt_map_
->end()) {
747 equals
= opt_info
.AreEqualByName(config_options
, opt_name
, this_ptr
,
751 if (equals
) { // False alarm, clear mismatch
755 if (equals
&& opt_info
.IsConfigurable() && opt_map_
!= nullptr) {
756 const auto* this_config
= opt_info
.AsRawPointer
<Configurable
>(this_ptr
);
757 if (this_config
== nullptr) {
758 const auto& iter
= opt_map_
->find(opt_name
);
759 // If the name exists in the map and is not empty/null,
760 // then the this_config should be set.
761 if (iter
!= opt_map_
->end() && !iter
->second
.empty() &&
762 iter
->second
!= kNullptrString
) {
763 *mismatch
= opt_name
;
772 ColumnFamilyOptions immutable_
;
773 ColumnFamilyOptions cf_options_
;
774 const std::unordered_map
<std::string
, std::string
>* opt_map_
;
777 std::unique_ptr
<Configurable
> CFOptionsAsConfigurable(
778 const MutableCFOptions
& opts
) {
779 std::unique_ptr
<Configurable
> ptr(new ConfigurableMutableCFOptions(opts
));
782 std::unique_ptr
<Configurable
> CFOptionsAsConfigurable(
783 const ColumnFamilyOptions
& opts
,
784 const std::unordered_map
<std::string
, std::string
>* opt_map
) {
785 std::unique_ptr
<Configurable
> ptr(new ConfigurableCFOptions(opts
, opt_map
));
788 #endif // ROCKSDB_LITE
790 ImmutableCFOptions::ImmutableCFOptions(const Options
& options
)
791 : ImmutableCFOptions(ImmutableDBOptions(options
), options
) {}
793 ImmutableCFOptions::ImmutableCFOptions(const ImmutableDBOptions
& db_options
,
794 const ColumnFamilyOptions
& cf_options
)
795 : compaction_style(cf_options
.compaction_style
),
796 compaction_pri(cf_options
.compaction_pri
),
797 user_comparator(cf_options
.comparator
),
798 internal_comparator(InternalKeyComparator(cf_options
.comparator
)),
799 merge_operator(cf_options
.merge_operator
.get()),
800 compaction_filter(cf_options
.compaction_filter
),
801 compaction_filter_factory(cf_options
.compaction_filter_factory
.get()),
802 min_write_buffer_number_to_merge(
803 cf_options
.min_write_buffer_number_to_merge
),
804 max_write_buffer_number_to_maintain(
805 cf_options
.max_write_buffer_number_to_maintain
),
806 max_write_buffer_size_to_maintain(
807 cf_options
.max_write_buffer_size_to_maintain
),
808 inplace_update_support(cf_options
.inplace_update_support
),
809 inplace_callback(cf_options
.inplace_callback
),
810 info_log(db_options
.info_log
.get()),
811 statistics(db_options
.statistics
.get()),
812 rate_limiter(db_options
.rate_limiter
.get()),
813 info_log_level(db_options
.info_log_level
),
815 fs(db_options
.fs
.get()),
816 allow_mmap_reads(db_options
.allow_mmap_reads
),
817 allow_mmap_writes(db_options
.allow_mmap_writes
),
818 db_paths(db_options
.db_paths
),
819 memtable_factory(cf_options
.memtable_factory
.get()),
820 table_factory(cf_options
.table_factory
.get()),
821 table_properties_collector_factories(
822 cf_options
.table_properties_collector_factories
),
823 advise_random_on_open(db_options
.advise_random_on_open
),
824 bloom_locality(cf_options
.bloom_locality
),
825 purge_redundant_kvs_while_flush(
826 cf_options
.purge_redundant_kvs_while_flush
),
827 use_fsync(db_options
.use_fsync
),
828 compression_per_level(cf_options
.compression_per_level
),
829 level_compaction_dynamic_level_bytes(
830 cf_options
.level_compaction_dynamic_level_bytes
),
831 access_hint_on_compaction_start(
832 db_options
.access_hint_on_compaction_start
),
833 new_table_reader_for_compaction_inputs(
834 db_options
.new_table_reader_for_compaction_inputs
),
835 num_levels(cf_options
.num_levels
),
836 optimize_filters_for_hits(cf_options
.optimize_filters_for_hits
),
837 force_consistency_checks(cf_options
.force_consistency_checks
),
838 allow_ingest_behind(db_options
.allow_ingest_behind
),
839 preserve_deletes(db_options
.preserve_deletes
),
840 listeners(db_options
.listeners
),
841 row_cache(db_options
.row_cache
),
842 memtable_insert_with_hint_prefix_extractor(
843 cf_options
.memtable_insert_with_hint_prefix_extractor
.get()),
844 cf_paths(cf_options
.cf_paths
),
845 compaction_thread_limiter(cf_options
.compaction_thread_limiter
),
846 file_checksum_gen_factory(db_options
.file_checksum_gen_factory
.get()),
847 sst_partitioner_factory(cf_options
.sst_partitioner_factory
),
848 allow_data_in_errors(db_options
.allow_data_in_errors
),
849 db_host_id(db_options
.db_host_id
) {}
851 // Multiple two operands. If they overflow, return op1.
852 uint64_t MultiplyCheckOverflow(uint64_t op1
, double op2
) {
853 if (op1
== 0 || op2
<= 0) {
856 if (port::kMaxUint64
/ op1
< op2
) {
859 return static_cast<uint64_t>(op1
* op2
);
862 // when level_compaction_dynamic_level_bytes is true and leveled compaction
863 // is used, the base level is not always L1, so precomupted max_file_size can
864 // no longer be used. Recompute file_size_for_level from base level.
865 uint64_t MaxFileSizeForLevel(const MutableCFOptions
& cf_options
,
866 int level
, CompactionStyle compaction_style
, int base_level
,
867 bool level_compaction_dynamic_level_bytes
) {
868 if (!level_compaction_dynamic_level_bytes
|| level
< base_level
||
869 compaction_style
!= kCompactionStyleLevel
) {
871 assert(level
< (int)cf_options
.max_file_size
.size());
872 return cf_options
.max_file_size
[level
];
874 assert(level
>= 0 && base_level
>= 0);
875 assert(level
- base_level
< (int)cf_options
.max_file_size
.size());
876 return cf_options
.max_file_size
[level
- base_level
];
880 size_t MaxFileSizeForL0MetaPin(const MutableCFOptions
& cf_options
) {
881 // We do not want to pin meta-blocks that almost certainly came from intra-L0
882 // or a former larger `write_buffer_size` value to avoid surprising users with
883 // pinned memory usage. We use a factor of 1.5 to account for overhead
884 // introduced during flush in most cases.
885 if (port::kMaxSizet
/ 3 < cf_options
.write_buffer_size
/ 2) {
886 return port::kMaxSizet
;
888 return cf_options
.write_buffer_size
/ 2 * 3;
891 void MutableCFOptions::RefreshDerivedOptions(int num_levels
,
892 CompactionStyle compaction_style
) {
893 max_file_size
.resize(num_levels
);
894 for (int i
= 0; i
< num_levels
; ++i
) {
895 if (i
== 0 && compaction_style
== kCompactionStyleUniversal
) {
896 max_file_size
[i
] = ULLONG_MAX
;
898 max_file_size
[i
] = MultiplyCheckOverflow(max_file_size
[i
- 1],
899 target_file_size_multiplier
);
901 max_file_size
[i
] = target_file_size_base
;
906 void MutableCFOptions::Dump(Logger
* log
) const {
907 // Memtable related options
909 " write_buffer_size: %" ROCKSDB_PRIszt
,
911 ROCKS_LOG_INFO(log
, " max_write_buffer_number: %d",
912 max_write_buffer_number
);
914 " arena_block_size: %" ROCKSDB_PRIszt
,
916 ROCKS_LOG_INFO(log
, " memtable_prefix_bloom_ratio: %f",
917 memtable_prefix_bloom_size_ratio
);
918 ROCKS_LOG_INFO(log
, " memtable_whole_key_filtering: %d",
919 memtable_whole_key_filtering
);
921 " memtable_huge_page_size: %" ROCKSDB_PRIszt
,
922 memtable_huge_page_size
);
924 " max_successive_merges: %" ROCKSDB_PRIszt
,
925 max_successive_merges
);
927 " inplace_update_num_locks: %" ROCKSDB_PRIszt
,
928 inplace_update_num_locks
);
930 log
, " prefix_extractor: %s",
931 prefix_extractor
== nullptr ? "nullptr" : prefix_extractor
->Name());
932 ROCKS_LOG_INFO(log
, " disable_auto_compactions: %d",
933 disable_auto_compactions
);
934 ROCKS_LOG_INFO(log
, " soft_pending_compaction_bytes_limit: %" PRIu64
,
935 soft_pending_compaction_bytes_limit
);
936 ROCKS_LOG_INFO(log
, " hard_pending_compaction_bytes_limit: %" PRIu64
,
937 hard_pending_compaction_bytes_limit
);
938 ROCKS_LOG_INFO(log
, " level0_file_num_compaction_trigger: %d",
939 level0_file_num_compaction_trigger
);
940 ROCKS_LOG_INFO(log
, " level0_slowdown_writes_trigger: %d",
941 level0_slowdown_writes_trigger
);
942 ROCKS_LOG_INFO(log
, " level0_stop_writes_trigger: %d",
943 level0_stop_writes_trigger
);
944 ROCKS_LOG_INFO(log
, " max_compaction_bytes: %" PRIu64
,
945 max_compaction_bytes
);
946 ROCKS_LOG_INFO(log
, " target_file_size_base: %" PRIu64
,
947 target_file_size_base
);
948 ROCKS_LOG_INFO(log
, " target_file_size_multiplier: %d",
949 target_file_size_multiplier
);
950 ROCKS_LOG_INFO(log
, " max_bytes_for_level_base: %" PRIu64
,
951 max_bytes_for_level_base
);
952 ROCKS_LOG_INFO(log
, " max_bytes_for_level_multiplier: %f",
953 max_bytes_for_level_multiplier
);
954 ROCKS_LOG_INFO(log
, " ttl: %" PRIu64
,
956 ROCKS_LOG_INFO(log
, " periodic_compaction_seconds: %" PRIu64
,
957 periodic_compaction_seconds
);
960 for (const auto m
: max_bytes_for_level_multiplier_additional
) {
961 snprintf(buf
, sizeof(buf
), "%d, ", m
);
964 if (result
.size() >= 2) {
965 result
.resize(result
.size() - 2);
970 ROCKS_LOG_INFO(log
, "max_bytes_for_level_multiplier_additional: %s",
972 ROCKS_LOG_INFO(log
, " max_sequential_skip_in_iterations: %" PRIu64
,
973 max_sequential_skip_in_iterations
);
974 ROCKS_LOG_INFO(log
, " check_flush_compaction_key_order: %d",
975 check_flush_compaction_key_order
);
976 ROCKS_LOG_INFO(log
, " paranoid_file_checks: %d",
977 paranoid_file_checks
);
978 ROCKS_LOG_INFO(log
, " report_bg_io_stats: %d",
980 ROCKS_LOG_INFO(log
, " compression: %d",
981 static_cast<int>(compression
));
983 // Universal Compaction Options
984 ROCKS_LOG_INFO(log
, "compaction_options_universal.size_ratio : %d",
985 compaction_options_universal
.size_ratio
);
986 ROCKS_LOG_INFO(log
, "compaction_options_universal.min_merge_width : %d",
987 compaction_options_universal
.min_merge_width
);
988 ROCKS_LOG_INFO(log
, "compaction_options_universal.max_merge_width : %d",
989 compaction_options_universal
.max_merge_width
);
991 log
, "compaction_options_universal.max_size_amplification_percent : %d",
992 compaction_options_universal
.max_size_amplification_percent
);
994 "compaction_options_universal.compression_size_percent : %d",
995 compaction_options_universal
.compression_size_percent
);
996 ROCKS_LOG_INFO(log
, "compaction_options_universal.stop_style : %d",
997 compaction_options_universal
.stop_style
);
999 log
, "compaction_options_universal.allow_trivial_move : %d",
1000 static_cast<int>(compaction_options_universal
.allow_trivial_move
));
1002 // FIFO Compaction Options
1003 ROCKS_LOG_INFO(log
, "compaction_options_fifo.max_table_files_size : %" PRIu64
,
1004 compaction_options_fifo
.max_table_files_size
);
1005 ROCKS_LOG_INFO(log
, "compaction_options_fifo.allow_compaction : %d",
1006 compaction_options_fifo
.allow_compaction
);
1008 // Blob file related options
1009 ROCKS_LOG_INFO(log
, " enable_blob_files: %s",
1010 enable_blob_files
? "true" : "false");
1011 ROCKS_LOG_INFO(log
, " min_blob_size: %" PRIu64
,
1013 ROCKS_LOG_INFO(log
, " blob_file_size: %" PRIu64
,
1015 ROCKS_LOG_INFO(log
, " blob_compression_type: %s",
1016 CompressionTypeToString(blob_compression_type
).c_str());
1017 ROCKS_LOG_INFO(log
, " enable_blob_garbage_collection: %s",
1018 enable_blob_garbage_collection
? "true" : "false");
1019 ROCKS_LOG_INFO(log
, " blob_garbage_collection_age_cutoff: %f",
1020 blob_garbage_collection_age_cutoff
);
1023 MutableCFOptions::MutableCFOptions(const Options
& options
)
1024 : MutableCFOptions(ColumnFamilyOptions(options
)) {}
1026 } // namespace ROCKSDB_NAMESPACE