]>
Commit | Line | Data |
---|---|---|
7c673cae | 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. |
11fdf7f2 TL |
2 | // This source code is licensed under both the GPLv2 (found in the |
3 | // COPYING file in the root directory) and Apache 2.0 License | |
4 | // (found in the LICENSE.Apache file in the root directory). | |
7c673cae FG |
5 | package org.rocksdb; |
6 | ||
7 | /** | |
8 | * The config for plain table sst format. | |
9 | * | |
10 | * BlockBasedTable is a RocksDB's default SST file format. | |
11 | */ | |
494da23a | 12 | //TODO(AR) should be renamed BlockBasedTableOptions |
7c673cae FG |
13 | public class BlockBasedTableConfig extends TableFormatConfig { |
14 | ||
15 | public BlockBasedTableConfig() { | |
494da23a TL |
16 | //TODO(AR) flushBlockPolicyFactory |
17 | cacheIndexAndFilterBlocks = false; | |
20effc67 | 18 | cacheIndexAndFilterBlocksWithHighPriority = true; |
494da23a TL |
19 | pinL0FilterAndIndexBlocksInCache = false; |
20 | pinTopLevelIndexAndFilter = true; | |
21 | indexType = IndexType.kBinarySearch; | |
22 | dataBlockIndexType = DataBlockIndexType.kDataBlockBinarySearch; | |
23 | dataBlockHashTableUtilRatio = 0.75; | |
24 | checksumType = ChecksumType.kCRC32c; | |
25 | noBlockCache = false; | |
26 | blockCache = null; | |
27 | persistentCache = null; | |
28 | blockCacheCompressed = null; | |
29 | blockSize = 4 * 1024; | |
30 | blockSizeDeviation = 10; | |
31 | blockRestartInterval = 16; | |
32 | indexBlockRestartInterval = 1; | |
33 | metadataBlockSize = 4096; | |
34 | partitionFilters = false; | |
20effc67 | 35 | optimizeFiltersForMemory = false; |
494da23a TL |
36 | useDeltaEncoding = true; |
37 | filterPolicy = null; | |
38 | wholeKeyFiltering = true; | |
20effc67 | 39 | verifyCompression = false; |
494da23a | 40 | readAmpBytesPerBit = 0; |
1e59de90 | 41 | formatVersion = 5; |
494da23a TL |
42 | enableIndexCompression = true; |
43 | blockAlign = false; | |
20effc67 | 44 | indexShortening = IndexShorteningMode.kShortenSeparators; |
494da23a TL |
45 | |
46 | // NOTE: ONLY used if blockCache == null | |
47 | blockCacheSize = 8 * 1024 * 1024; | |
48 | blockCacheNumShardBits = 0; | |
49 | ||
50 | // NOTE: ONLY used if blockCacheCompressed == null | |
51 | blockCacheCompressedSize = 0; | |
52 | blockCacheCompressedNumShardBits = 0; | |
7c673cae FG |
53 | } |
54 | ||
55 | /** | |
494da23a TL |
56 | * Indicating if we'd put index/filter blocks to the block cache. |
57 | * If not specified, each "table reader" object will pre-load index/filter | |
58 | * block during table initialization. | |
7c673cae | 59 | * |
494da23a TL |
60 | * @return if index and filter blocks should be put in block cache. |
61 | */ | |
62 | public boolean cacheIndexAndFilterBlocks() { | |
63 | return cacheIndexAndFilterBlocks; | |
64 | } | |
65 | ||
66 | /** | |
67 | * Indicating if we'd put index/filter blocks to the block cache. | |
68 | * If not specified, each "table reader" object will pre-load index/filter | |
69 | * block during table initialization. | |
70 | * | |
71 | * @param cacheIndexAndFilterBlocks and filter blocks should be put in block cache. | |
7c673cae FG |
72 | * @return the reference to the current config. |
73 | */ | |
494da23a TL |
74 | public BlockBasedTableConfig setCacheIndexAndFilterBlocks( |
75 | final boolean cacheIndexAndFilterBlocks) { | |
76 | this.cacheIndexAndFilterBlocks = cacheIndexAndFilterBlocks; | |
7c673cae FG |
77 | return this; |
78 | } | |
79 | ||
80 | /** | |
494da23a | 81 | * Indicates if index and filter blocks will be treated as high-priority in the block cache. |
20effc67 | 82 | * See note below about applicability. If not specified, defaults to true. |
494da23a TL |
83 | * |
84 | * @return if index and filter blocks will be treated as high-priority. | |
7c673cae | 85 | */ |
494da23a TL |
86 | public boolean cacheIndexAndFilterBlocksWithHighPriority() { |
87 | return cacheIndexAndFilterBlocksWithHighPriority; | |
7c673cae FG |
88 | } |
89 | ||
90 | /** | |
494da23a TL |
91 | * If true, cache index and filter blocks with high priority. If set to true, |
92 | * depending on implementation of block cache, index and filter blocks may be | |
93 | * less likely to be evicted than data blocks. | |
7c673cae | 94 | * |
494da23a TL |
95 | * @param cacheIndexAndFilterBlocksWithHighPriority if index and filter blocks |
96 | * will be treated as high-priority. | |
7c673cae FG |
97 | * @return the reference to the current config. |
98 | */ | |
494da23a TL |
99 | public BlockBasedTableConfig setCacheIndexAndFilterBlocksWithHighPriority( |
100 | final boolean cacheIndexAndFilterBlocksWithHighPriority) { | |
101 | this.cacheIndexAndFilterBlocksWithHighPriority = cacheIndexAndFilterBlocksWithHighPriority; | |
7c673cae FG |
102 | return this; |
103 | } | |
104 | ||
105 | /** | |
494da23a TL |
106 | * Indicating if we'd like to pin L0 index/filter blocks to the block cache. |
107 | If not specified, defaults to false. | |
108 | * | |
109 | * @return if L0 index and filter blocks should be pinned to the block cache. | |
7c673cae | 110 | */ |
494da23a TL |
111 | public boolean pinL0FilterAndIndexBlocksInCache() { |
112 | return pinL0FilterAndIndexBlocksInCache; | |
113 | } | |
114 | ||
115 | /** | |
116 | * Indicating if we'd like to pin L0 index/filter blocks to the block cache. | |
117 | If not specified, defaults to false. | |
118 | * | |
119 | * @param pinL0FilterAndIndexBlocksInCache pin blocks in block cache | |
120 | * @return the reference to the current config. | |
121 | */ | |
122 | public BlockBasedTableConfig setPinL0FilterAndIndexBlocksInCache( | |
123 | final boolean pinL0FilterAndIndexBlocksInCache) { | |
124 | this.pinL0FilterAndIndexBlocksInCache = pinL0FilterAndIndexBlocksInCache; | |
125 | return this; | |
126 | } | |
127 | ||
128 | /** | |
129 | * Indicates if top-level index and filter blocks should be pinned. | |
130 | * | |
131 | * @return if top-level index and filter blocks should be pinned. | |
132 | */ | |
133 | public boolean pinTopLevelIndexAndFilter() { | |
134 | return pinTopLevelIndexAndFilter; | |
135 | } | |
136 | ||
137 | /** | |
138 | * If cacheIndexAndFilterBlocks is true and the below is true, then | |
139 | * the top-level index of partitioned filter and index blocks are stored in | |
140 | * the cache, but a reference is held in the "table reader" object so the | |
141 | * blocks are pinned and only evicted from cache when the table reader is | |
142 | * freed. This is not limited to l0 in LSM tree. | |
143 | * | |
144 | * @param pinTopLevelIndexAndFilter if top-level index and filter blocks should be pinned. | |
145 | * @return the reference to the current config. | |
146 | */ | |
147 | public BlockBasedTableConfig setPinTopLevelIndexAndFilter(final boolean pinTopLevelIndexAndFilter) { | |
148 | this.pinTopLevelIndexAndFilter = pinTopLevelIndexAndFilter; | |
149 | return this; | |
150 | } | |
151 | ||
152 | /** | |
153 | * Get the index type. | |
154 | * | |
155 | * @return the currently set index type | |
156 | */ | |
157 | public IndexType indexType() { | |
158 | return indexType; | |
159 | } | |
160 | ||
161 | /** | |
162 | * Sets the index type to used with this table. | |
163 | * | |
164 | * @param indexType {@link org.rocksdb.IndexType} value | |
165 | * @return the reference to the current option. | |
166 | */ | |
167 | public BlockBasedTableConfig setIndexType( | |
168 | final IndexType indexType) { | |
169 | this.indexType = indexType; | |
170 | return this; | |
171 | } | |
172 | ||
173 | /** | |
174 | * Get the data block index type. | |
175 | * | |
176 | * @return the currently set data block index type | |
177 | */ | |
178 | public DataBlockIndexType dataBlockIndexType() { | |
179 | return dataBlockIndexType; | |
180 | } | |
181 | ||
182 | /** | |
183 | * Sets the data block index type to used with this table. | |
184 | * | |
185 | * @param dataBlockIndexType {@link org.rocksdb.DataBlockIndexType} value | |
186 | * @return the reference to the current option. | |
187 | */ | |
188 | public BlockBasedTableConfig setDataBlockIndexType( | |
189 | final DataBlockIndexType dataBlockIndexType) { | |
190 | this.dataBlockIndexType = dataBlockIndexType; | |
191 | return this; | |
192 | } | |
193 | ||
194 | /** | |
195 | * Get the #entries/#buckets. It is valid only when {@link #dataBlockIndexType()} is | |
196 | * {@link DataBlockIndexType#kDataBlockBinaryAndHash}. | |
197 | * | |
198 | * @return the #entries/#buckets. | |
199 | */ | |
200 | public double dataBlockHashTableUtilRatio() { | |
201 | return dataBlockHashTableUtilRatio; | |
202 | } | |
203 | ||
204 | /** | |
205 | * Set the #entries/#buckets. It is valid only when {@link #dataBlockIndexType()} is | |
206 | * {@link DataBlockIndexType#kDataBlockBinaryAndHash}. | |
207 | * | |
208 | * @param dataBlockHashTableUtilRatio #entries/#buckets | |
209 | * @return the reference to the current option. | |
210 | */ | |
211 | public BlockBasedTableConfig setDataBlockHashTableUtilRatio( | |
212 | final double dataBlockHashTableUtilRatio) { | |
213 | this.dataBlockHashTableUtilRatio = dataBlockHashTableUtilRatio; | |
214 | return this; | |
215 | } | |
216 | ||
217 | /** | |
218 | * Get the checksum type to be used with this table. | |
219 | * | |
220 | * @return the currently set checksum type | |
221 | */ | |
222 | public ChecksumType checksumType() { | |
223 | return checksumType; | |
224 | } | |
225 | ||
226 | /** | |
227 | * Sets | |
228 | * | |
229 | * @param checksumType {@link org.rocksdb.ChecksumType} value. | |
230 | * @return the reference to the current option. | |
231 | */ | |
232 | public BlockBasedTableConfig setChecksumType( | |
233 | final ChecksumType checksumType) { | |
234 | this.checksumType = checksumType; | |
235 | return this; | |
236 | } | |
237 | ||
238 | /** | |
239 | * Determine if the block cache is disabled. | |
240 | * | |
241 | * @return if block cache is disabled | |
242 | */ | |
243 | public boolean noBlockCache() { | |
244 | return noBlockCache; | |
245 | } | |
246 | ||
247 | /** | |
248 | * Disable block cache. If this is set to true, | |
249 | * then no block cache should be used, and the {@link #setBlockCache(Cache)} | |
250 | * should point to a {@code null} object. | |
251 | * | |
252 | * Default: false | |
253 | * | |
254 | * @param noBlockCache if use block cache | |
255 | * @return the reference to the current config. | |
256 | */ | |
257 | public BlockBasedTableConfig setNoBlockCache(final boolean noBlockCache) { | |
258 | this.noBlockCache = noBlockCache; | |
259 | return this; | |
7c673cae FG |
260 | } |
261 | ||
11fdf7f2 TL |
262 | /** |
263 | * Use the specified cache for blocks. | |
264 | * When not null this take precedence even if the user sets a block cache size. | |
265 | * | |
266 | * {@link org.rocksdb.Cache} should not be disposed before options instances | |
267 | * using this cache is disposed. | |
268 | * | |
269 | * {@link org.rocksdb.Cache} instance can be re-used in multiple options | |
270 | * instances. | |
271 | * | |
494da23a TL |
272 | * @param blockCache {@link org.rocksdb.Cache} Cache java instance |
273 | * (e.g. LRUCache). | |
274 | * | |
11fdf7f2 TL |
275 | * @return the reference to the current config. |
276 | */ | |
494da23a TL |
277 | public BlockBasedTableConfig setBlockCache(final Cache blockCache) { |
278 | this.blockCache = blockCache; | |
11fdf7f2 TL |
279 | return this; |
280 | } | |
281 | ||
7c673cae | 282 | /** |
494da23a | 283 | * Use the specified persistent cache. |
7c673cae | 284 | * |
494da23a TL |
285 | * If {@code !null} use the specified cache for pages read from device, |
286 | * otherwise no page cache is used. | |
287 | * | |
288 | * @param persistentCache the persistent cache | |
289 | * | |
290 | * @return the reference to the current config. | |
7c673cae | 291 | */ |
494da23a TL |
292 | public BlockBasedTableConfig setPersistentCache( |
293 | final PersistentCache persistentCache) { | |
294 | this.persistentCache = persistentCache; | |
7c673cae FG |
295 | return this; |
296 | } | |
297 | ||
298 | /** | |
494da23a | 299 | * Use the specified cache for compressed blocks. |
7c673cae | 300 | * |
494da23a TL |
301 | * If {@code null}, RocksDB will not use a compressed block cache. |
302 | * | |
303 | * Note: though it looks similar to {@link #setBlockCache(Cache)}, RocksDB | |
304 | * doesn't put the same type of object there. | |
305 | * | |
306 | * {@link org.rocksdb.Cache} should not be disposed before options instances | |
307 | * using this cache is disposed. | |
308 | * | |
309 | * {@link org.rocksdb.Cache} instance can be re-used in multiple options | |
310 | * instances. | |
311 | * | |
312 | * @param blockCacheCompressed {@link org.rocksdb.Cache} Cache java instance | |
313 | * (e.g. LRUCache). | |
314 | * | |
315 | * @return the reference to the current config. | |
7c673cae | 316 | */ |
494da23a TL |
317 | public BlockBasedTableConfig setBlockCacheCompressed( |
318 | final Cache blockCacheCompressed) { | |
319 | this.blockCacheCompressed = blockCacheCompressed; | |
320 | return this; | |
321 | } | |
322 | ||
323 | /** | |
324 | * Get the approximate size of user data packed per block. | |
325 | * | |
326 | * @return block size in bytes | |
327 | */ | |
328 | public long blockSize() { | |
329 | return blockSize; | |
7c673cae FG |
330 | } |
331 | ||
332 | /** | |
494da23a | 333 | * Approximate size of user data packed per block. Note that the |
7c673cae FG |
334 | * block size specified here corresponds to uncompressed data. The |
335 | * actual size of the unit read from disk may be smaller if | |
336 | * compression is enabled. This parameter can be changed dynamically. | |
337 | * Default: 4K | |
338 | * | |
339 | * @param blockSize block size in bytes | |
340 | * @return the reference to the current config. | |
341 | */ | |
342 | public BlockBasedTableConfig setBlockSize(final long blockSize) { | |
494da23a | 343 | this.blockSize = blockSize; |
7c673cae FG |
344 | return this; |
345 | } | |
346 | ||
347 | /** | |
494da23a | 348 | * @return the hash table ratio. |
7c673cae | 349 | */ |
494da23a TL |
350 | public int blockSizeDeviation() { |
351 | return blockSizeDeviation; | |
7c673cae FG |
352 | } |
353 | ||
354 | /** | |
355 | * This is used to close a block before it reaches the configured | |
494da23a TL |
356 | * {@link #blockSize()}. If the percentage of free space in the current block |
357 | * is less than this specified number and adding a new record to the block | |
358 | * will exceed the configured block size, then this block will be closed and | |
359 | * the new record will be written to the next block. | |
360 | * | |
7c673cae FG |
361 | * Default is 10. |
362 | * | |
363 | * @param blockSizeDeviation the deviation to block size allowed | |
364 | * @return the reference to the current config. | |
365 | */ | |
366 | public BlockBasedTableConfig setBlockSizeDeviation( | |
367 | final int blockSizeDeviation) { | |
494da23a | 368 | this.blockSizeDeviation = blockSizeDeviation; |
7c673cae FG |
369 | return this; |
370 | } | |
371 | ||
372 | /** | |
494da23a TL |
373 | * Get the block restart interval. |
374 | * | |
375 | * @return block restart interval | |
7c673cae | 376 | */ |
494da23a TL |
377 | public int blockRestartInterval() { |
378 | return blockRestartInterval; | |
7c673cae FG |
379 | } |
380 | ||
381 | /** | |
494da23a | 382 | * Set the block restart interval. |
7c673cae FG |
383 | * |
384 | * @param restartInterval block restart interval. | |
385 | * @return the reference to the current config. | |
386 | */ | |
387 | public BlockBasedTableConfig setBlockRestartInterval( | |
388 | final int restartInterval) { | |
494da23a | 389 | blockRestartInterval = restartInterval; |
7c673cae FG |
390 | return this; |
391 | } | |
392 | ||
393 | /** | |
494da23a TL |
394 | * Get the index block restart interval. |
395 | * | |
396 | * @return index block restart interval | |
7c673cae | 397 | */ |
494da23a TL |
398 | public int indexBlockRestartInterval() { |
399 | return indexBlockRestartInterval; | |
7c673cae FG |
400 | } |
401 | ||
402 | /** | |
494da23a | 403 | * Set the index block restart interval |
7c673cae | 404 | * |
494da23a | 405 | * @param restartInterval index block restart interval. |
7c673cae FG |
406 | * @return the reference to the current config. |
407 | */ | |
494da23a TL |
408 | public BlockBasedTableConfig setIndexBlockRestartInterval( |
409 | final int restartInterval) { | |
410 | indexBlockRestartInterval = restartInterval; | |
7c673cae FG |
411 | return this; |
412 | } | |
413 | ||
414 | /** | |
494da23a TL |
415 | * Get the block size for partitioned metadata. |
416 | * | |
417 | * @return block size for partitioned metadata. | |
7c673cae | 418 | */ |
494da23a TL |
419 | public long metadataBlockSize() { |
420 | return metadataBlockSize; | |
421 | } | |
422 | ||
423 | /** | |
424 | * Set block size for partitioned metadata. | |
425 | * | |
426 | * @param metadataBlockSize Partitioned metadata block size. | |
427 | * @return the reference to the current config. | |
428 | */ | |
429 | public BlockBasedTableConfig setMetadataBlockSize( | |
430 | final long metadataBlockSize) { | |
431 | this.metadataBlockSize = metadataBlockSize; | |
432 | return this; | |
433 | } | |
434 | ||
435 | /** | |
436 | * Indicates if we're using partitioned filters. | |
437 | * | |
438 | * @return if we're using partition filters. | |
439 | */ | |
440 | public boolean partitionFilters() { | |
441 | return partitionFilters; | |
442 | } | |
443 | ||
444 | /** | |
445 | * Use partitioned full filters for each SST file. This option is incompatible | |
446 | * with block-based filters. | |
447 | * | |
448 | * Defaults to false. | |
449 | * | |
450 | * @param partitionFilters use partition filters. | |
451 | * @return the reference to the current config. | |
452 | */ | |
453 | public BlockBasedTableConfig setPartitionFilters(final boolean partitionFilters) { | |
454 | this.partitionFilters = partitionFilters; | |
455 | return this; | |
456 | } | |
457 | ||
20effc67 TL |
458 | /*** |
459 | * Option to generate Bloom filters that minimize memory | |
460 | * internal fragmentation. | |
461 | * | |
462 | * See {@link #setOptimizeFiltersForMemory(boolean)}. | |
463 | * | |
464 | * @return true if bloom filters are used to minimize memory internal | |
465 | * fragmentation | |
466 | */ | |
467 | @Experimental("Option to generate Bloom filters that minimize memory internal fragmentation") | |
468 | public boolean optimizeFiltersForMemory() { | |
469 | return optimizeFiltersForMemory; | |
470 | } | |
471 | ||
472 | /** | |
473 | * Option to generate Bloom filters that minimize memory | |
474 | * internal fragmentation. | |
475 | * | |
476 | * When false, malloc_usable_size is not available, or format_version < 5, | |
477 | * filters are generated without regard to internal fragmentation when | |
478 | * loaded into memory (historical behavior). When true (and | |
479 | * malloc_usable_size is available and {@link #formatVersion()} >= 5), | |
480 | * then Bloom filters are generated to "round up" and "round down" their | |
481 | * sizes to minimize internal fragmentation when loaded into memory, assuming | |
482 | * the reading DB has the same memory allocation characteristics as the | |
483 | * generating DB. This option does not break forward or backward | |
484 | * compatibility. | |
485 | * | |
486 | * While individual filters will vary in bits/key and false positive rate | |
487 | * when setting is true, the implementation attempts to maintain a weighted | |
488 | * average FP rate for filters consistent with this option set to false. | |
489 | * | |
490 | * With Jemalloc for example, this setting is expected to save about 10% of | |
491 | * the memory footprint and block cache charge of filters, while increasing | |
492 | * disk usage of filters by about 1-2% due to encoding efficiency losses | |
493 | * with variance in bits/key. | |
494 | * | |
495 | * NOTE: Because some memory counted by block cache might be unmapped pages | |
496 | * within internal fragmentation, this option can increase observed RSS | |
497 | * memory usage. With {@link #cacheIndexAndFilterBlocks()} == true, | |
498 | * this option makes the block cache better at using space it is allowed. | |
499 | * | |
500 | * NOTE: Do not set to true if you do not trust malloc_usable_size. With | |
501 | * this option, RocksDB might access an allocated memory object beyond its | |
502 | * original size if malloc_usable_size says it is safe to do so. While this | |
503 | * can be considered bad practice, it should not produce undefined behavior | |
504 | * unless malloc_usable_size is buggy or broken. | |
505 | * | |
506 | * @param optimizeFiltersForMemory true to enable Bloom filters that minimize | |
507 | * memory internal fragmentation, or false to disable. | |
508 | * | |
509 | * @return the reference to the current config. | |
510 | */ | |
511 | @Experimental("Option to generate Bloom filters that minimize memory internal fragmentation") | |
512 | public BlockBasedTableConfig setOptimizeFiltersForMemory(final boolean optimizeFiltersForMemory) { | |
513 | this.optimizeFiltersForMemory = optimizeFiltersForMemory; | |
514 | return this; | |
515 | } | |
516 | ||
494da23a TL |
517 | /** |
518 | * Determine if delta encoding is being used to compress block keys. | |
519 | * | |
520 | * @return true if delta encoding is enabled, false otherwise. | |
521 | */ | |
522 | public boolean useDeltaEncoding() { | |
523 | return useDeltaEncoding; | |
524 | } | |
525 | ||
526 | /** | |
527 | * Use delta encoding to compress keys in blocks. | |
528 | * | |
529 | * NOTE: {@link ReadOptions#pinData()} requires this option to be disabled. | |
530 | * | |
531 | * Default: true | |
532 | * | |
533 | * @param useDeltaEncoding true to enable delta encoding | |
534 | * | |
535 | * @return the reference to the current config. | |
536 | */ | |
537 | public BlockBasedTableConfig setUseDeltaEncoding( | |
538 | final boolean useDeltaEncoding) { | |
539 | this.useDeltaEncoding = useDeltaEncoding; | |
540 | return this; | |
7c673cae FG |
541 | } |
542 | ||
f67539c2 TL |
543 | /** |
544 | * Get the filter policy. | |
545 | * | |
546 | * @return the current filter policy. | |
547 | */ | |
548 | public Filter filterPolicy() { | |
549 | return filterPolicy; | |
550 | } | |
551 | ||
7c673cae FG |
552 | /** |
553 | * Use the specified filter policy to reduce disk reads. | |
554 | * | |
1e59de90 TL |
555 | * {@link org.rocksdb.Filter} should not be closed before options instances |
556 | * using this filter are closed. | |
7c673cae FG |
557 | * |
558 | * {@link org.rocksdb.Filter} instance can be re-used in multiple options | |
559 | * instances. | |
560 | * | |
494da23a | 561 | * @param filterPolicy {@link org.rocksdb.Filter} Filter Policy java instance. |
7c673cae FG |
562 | * @return the reference to the current config. |
563 | */ | |
494da23a TL |
564 | public BlockBasedTableConfig setFilterPolicy( |
565 | final Filter filterPolicy) { | |
566 | this.filterPolicy = filterPolicy; | |
567 | return this; | |
568 | } | |
569 | ||
f67539c2 TL |
570 | /** |
571 | * Set the filter. | |
572 | * | |
573 | * @param filter the filter | |
574 | * @return the reference to the current config. | |
575 | * | |
494da23a TL |
576 | * @deprecated Use {@link #setFilterPolicy(Filter)} |
577 | */ | |
578 | @Deprecated | |
7c673cae FG |
579 | public BlockBasedTableConfig setFilter( |
580 | final Filter filter) { | |
494da23a TL |
581 | return setFilterPolicy(filter); |
582 | } | |
583 | ||
584 | /** | |
585 | * Determine if whole keys as opposed to prefixes are placed in the filter. | |
586 | * | |
587 | * @return if whole key filtering is enabled | |
588 | */ | |
589 | public boolean wholeKeyFiltering() { | |
590 | return wholeKeyFiltering; | |
591 | } | |
592 | ||
593 | /** | |
594 | * If true, place whole keys in the filter (not just prefixes). | |
595 | * This must generally be true for gets to be efficient. | |
596 | * Default: true | |
597 | * | |
598 | * @param wholeKeyFiltering if enable whole key filtering | |
599 | * @return the reference to the current config. | |
600 | */ | |
601 | public BlockBasedTableConfig setWholeKeyFiltering( | |
602 | final boolean wholeKeyFiltering) { | |
603 | this.wholeKeyFiltering = wholeKeyFiltering; | |
7c673cae FG |
604 | return this; |
605 | } | |
606 | ||
607 | /** | |
494da23a | 608 | * Returns true when compression verification is enabled. |
7c673cae | 609 | * |
494da23a TL |
610 | * See {@link #setVerifyCompression(boolean)}. |
611 | * | |
612 | * @return true if compression verification is enabled. | |
7c673cae | 613 | */ |
494da23a TL |
614 | public boolean verifyCompression() { |
615 | return verifyCompression; | |
7c673cae FG |
616 | } |
617 | ||
618 | /** | |
494da23a TL |
619 | * Verify that decompressing the compressed block gives back the input. This |
620 | * is a verification mode that we use to detect bugs in compression | |
621 | * algorithms. | |
622 | * | |
623 | * @param verifyCompression true to enable compression verification. | |
7c673cae | 624 | * |
7c673cae FG |
625 | * @return the reference to the current config. |
626 | */ | |
494da23a TL |
627 | public BlockBasedTableConfig setVerifyCompression( |
628 | final boolean verifyCompression) { | |
629 | this.verifyCompression = verifyCompression; | |
7c673cae FG |
630 | return this; |
631 | } | |
632 | ||
633 | /** | |
494da23a | 634 | * Get the Read amplification bytes per-bit. |
7c673cae | 635 | * |
494da23a TL |
636 | * See {@link #setReadAmpBytesPerBit(int)}. |
637 | * | |
638 | * @return the bytes per-bit. | |
7c673cae | 639 | */ |
494da23a TL |
640 | public int readAmpBytesPerBit() { |
641 | return readAmpBytesPerBit; | |
7c673cae FG |
642 | } |
643 | ||
644 | /** | |
494da23a TL |
645 | * Set the Read amplification bytes per-bit. |
646 | * | |
647 | * If used, For every data block we load into memory, we will create a bitmap | |
648 | * of size ((block_size / `read_amp_bytes_per_bit`) / 8) bytes. This bitmap | |
649 | * will be used to figure out the percentage we actually read of the blocks. | |
650 | * | |
651 | * When this feature is used Tickers::READ_AMP_ESTIMATE_USEFUL_BYTES and | |
652 | * Tickers::READ_AMP_TOTAL_READ_BYTES can be used to calculate the | |
653 | * read amplification using this formula | |
654 | * (READ_AMP_TOTAL_READ_BYTES / READ_AMP_ESTIMATE_USEFUL_BYTES) | |
655 | * | |
656 | * value => memory usage (percentage of loaded blocks memory) | |
657 | * 1 => 12.50 % | |
658 | * 2 => 06.25 % | |
659 | * 4 => 03.12 % | |
660 | * 8 => 01.56 % | |
661 | * 16 => 00.78 % | |
662 | * | |
663 | * Note: This number must be a power of 2, if not it will be sanitized | |
664 | * to be the next lowest power of 2, for example a value of 7 will be | |
665 | * treated as 4, a value of 19 will be treated as 16. | |
666 | * | |
667 | * Default: 0 (disabled) | |
668 | * | |
669 | * @param readAmpBytesPerBit the bytes per-bit | |
7c673cae | 670 | * |
7c673cae FG |
671 | * @return the reference to the current config. |
672 | */ | |
494da23a TL |
673 | public BlockBasedTableConfig setReadAmpBytesPerBit(final int readAmpBytesPerBit) { |
674 | this.readAmpBytesPerBit = readAmpBytesPerBit; | |
7c673cae FG |
675 | return this; |
676 | } | |
677 | ||
678 | /** | |
494da23a TL |
679 | * Get the format version. |
680 | * See {@link #setFormatVersion(int)}. | |
7c673cae | 681 | * |
494da23a | 682 | * @return the currently configured format version. |
7c673cae | 683 | */ |
494da23a TL |
684 | public int formatVersion() { |
685 | return formatVersion; | |
7c673cae FG |
686 | } |
687 | ||
688 | /** | |
494da23a | 689 | * <p>We currently have five versions:</p> |
7c673cae | 690 | * |
494da23a TL |
691 | * <ul> |
692 | * <li><strong>0</strong> - This version is currently written | |
693 | * out by all RocksDB's versions by default. Can be read by really old | |
694 | * RocksDB's. Doesn't support changing checksum (default is CRC32).</li> | |
695 | * <li><strong>1</strong> - Can be read by RocksDB's versions since 3.0. | |
696 | * Supports non-default checksum, like xxHash. It is written by RocksDB when | |
697 | * BlockBasedTableOptions::checksum is something other than kCRC32c. (version | |
698 | * 0 is silently upconverted)</li> | |
699 | * <li><strong>2</strong> - Can be read by RocksDB's versions since 3.10. | |
700 | * Changes the way we encode compressed blocks with LZ4, BZip2 and Zlib | |
701 | * compression. If you don't plan to run RocksDB before version 3.10, | |
702 | * you should probably use this.</li> | |
703 | * <li><strong>3</strong> - Can be read by RocksDB's versions since 5.15. Changes the way we | |
704 | * encode the keys in index blocks. If you don't plan to run RocksDB before | |
705 | * version 5.15, you should probably use this. | |
706 | * This option only affects newly written tables. When reading existing | |
707 | * tables, the information about version is read from the footer.</li> | |
708 | * <li><strong>4</strong> - Can be read by RocksDB's versions since 5.16. Changes the way we | |
709 | * encode the values in index blocks. If you don't plan to run RocksDB before | |
710 | * version 5.16 and you are using index_block_restart_interval > 1, you should | |
20effc67 TL |
711 | * probably use this as it would reduce the index size. |
712 | * This option only affects newly written tables. When reading existing | |
713 | * tables, the information about version is read from the footer.</li> | |
714 | * <li><strong>5</strong> - Can be read by RocksDB's versions since 6.6.0. | |
715 | * Full and partitioned filters use a generally faster and more accurate | |
716 | * Bloom filter implementation, with a different schema.</li> | |
494da23a | 717 | * </ul> |
494da23a TL |
718 | * |
719 | * @param formatVersion integer representing the version to be used. | |
720 | * | |
721 | * @return the reference to the current option. | |
722 | */ | |
723 | public BlockBasedTableConfig setFormatVersion( | |
724 | final int formatVersion) { | |
20effc67 | 725 | assert (formatVersion >= 0); |
494da23a TL |
726 | this.formatVersion = formatVersion; |
727 | return this; | |
728 | } | |
729 | ||
730 | /** | |
731 | * Determine if index compression is enabled. | |
732 | * | |
733 | * See {@link #setEnableIndexCompression(boolean)}. | |
734 | * | |
735 | * @return true if index compression is enabled, false otherwise | |
736 | */ | |
737 | public boolean enableIndexCompression() { | |
738 | return enableIndexCompression; | |
739 | } | |
740 | ||
741 | /** | |
742 | * Store index blocks on disk in compressed format. | |
743 | * | |
744 | * Changing this option to false will avoid the overhead of decompression | |
745 | * if index blocks are evicted and read back. | |
746 | * | |
747 | * @param enableIndexCompression true to enable index compression, | |
748 | * false to disable | |
749 | * | |
750 | * @return the reference to the current option. | |
751 | */ | |
752 | public BlockBasedTableConfig setEnableIndexCompression( | |
753 | final boolean enableIndexCompression) { | |
754 | this.enableIndexCompression = enableIndexCompression; | |
755 | return this; | |
756 | } | |
757 | ||
758 | /** | |
759 | * Determines whether data blocks are aligned on the lesser of page size | |
760 | * and block size. | |
761 | * | |
762 | * @return true if data blocks are aligned on the lesser of page size | |
763 | * and block size. | |
764 | */ | |
765 | public boolean blockAlign() { | |
766 | return blockAlign; | |
767 | } | |
768 | ||
769 | /** | |
770 | * Set whether data blocks should be aligned on the lesser of page size | |
771 | * and block size. | |
772 | * | |
773 | * @param blockAlign true to align data blocks on the lesser of page size | |
774 | * and block size. | |
775 | * | |
776 | * @return the reference to the current option. | |
777 | */ | |
778 | public BlockBasedTableConfig setBlockAlign(final boolean blockAlign) { | |
779 | this.blockAlign = blockAlign; | |
780 | return this; | |
781 | } | |
782 | ||
20effc67 TL |
783 | /** |
784 | * Get the index shortening mode. | |
785 | * | |
786 | * @return the index shortening mode. | |
787 | */ | |
788 | public IndexShorteningMode indexShortening() { | |
789 | return indexShortening; | |
790 | } | |
791 | ||
792 | /** | |
793 | * Set the index shortening mode. | |
794 | * | |
795 | * See {@link IndexShorteningMode}. | |
796 | * | |
797 | * @param indexShortening the index shortening mode. | |
798 | * | |
799 | * @return the reference to the current option. | |
800 | */ | |
801 | public BlockBasedTableConfig setIndexShortening(final IndexShorteningMode indexShortening) { | |
802 | this.indexShortening = indexShortening; | |
803 | return this; | |
804 | } | |
494da23a TL |
805 | |
806 | /** | |
807 | * Get the size of the cache in bytes that will be used by RocksDB. | |
808 | * | |
809 | * @return block cache size in bytes | |
810 | */ | |
811 | @Deprecated | |
812 | public long blockCacheSize() { | |
813 | return blockCacheSize; | |
814 | } | |
815 | ||
816 | /** | |
817 | * Set the size of the cache in bytes that will be used by RocksDB. | |
f67539c2 | 818 | * If cacheSize is negative, then cache will not be used. |
494da23a TL |
819 | * DEFAULT: 8M |
820 | * | |
821 | * @param blockCacheSize block cache size in bytes | |
7c673cae | 822 | * @return the reference to the current config. |
494da23a TL |
823 | * |
824 | * @deprecated Use {@link #setBlockCache(Cache)}. | |
7c673cae | 825 | */ |
494da23a TL |
826 | @Deprecated |
827 | public BlockBasedTableConfig setBlockCacheSize(final long blockCacheSize) { | |
828 | this.blockCacheSize = blockCacheSize; | |
829 | return this; | |
830 | } | |
831 | ||
832 | /** | |
833 | * Returns the number of shard bits used in the block cache. | |
834 | * The resulting number of shards would be 2 ^ (returned value). | |
835 | * Any negative number means use default settings. | |
836 | * | |
837 | * @return the number of shard bits used in the block cache. | |
838 | */ | |
839 | @Deprecated | |
840 | public int cacheNumShardBits() { | |
841 | return blockCacheNumShardBits; | |
842 | } | |
843 | ||
844 | /** | |
845 | * Controls the number of shards for the block cache. | |
846 | * This is applied only if cacheSize is set to non-negative. | |
847 | * | |
848 | * @param blockCacheNumShardBits the number of shard bits. The resulting | |
849 | * number of shards would be 2 ^ numShardBits. Any negative | |
850 | * number means use default settings." | |
851 | * @return the reference to the current option. | |
852 | * | |
853 | * @deprecated Use {@link #setBlockCache(Cache)}. | |
854 | */ | |
855 | @Deprecated | |
856 | public BlockBasedTableConfig setCacheNumShardBits( | |
857 | final int blockCacheNumShardBits) { | |
858 | this.blockCacheNumShardBits = blockCacheNumShardBits; | |
7c673cae FG |
859 | return this; |
860 | } | |
861 | ||
862 | /** | |
863 | * Size of compressed block cache. If 0, then block_cache_compressed is set | |
864 | * to null. | |
865 | * | |
866 | * @return size of compressed block cache. | |
867 | */ | |
494da23a | 868 | @Deprecated |
7c673cae | 869 | public long blockCacheCompressedSize() { |
494da23a | 870 | return blockCacheCompressedSize; |
7c673cae FG |
871 | } |
872 | ||
873 | /** | |
874 | * Size of compressed block cache. If 0, then block_cache_compressed is set | |
875 | * to null. | |
876 | * | |
877 | * @param blockCacheCompressedSize of compressed block cache. | |
878 | * @return the reference to the current config. | |
494da23a TL |
879 | * |
880 | * @deprecated Use {@link #setBlockCacheCompressed(Cache)}. | |
7c673cae | 881 | */ |
494da23a | 882 | @Deprecated |
7c673cae FG |
883 | public BlockBasedTableConfig setBlockCacheCompressedSize( |
884 | final long blockCacheCompressedSize) { | |
494da23a | 885 | this.blockCacheCompressedSize = blockCacheCompressedSize; |
7c673cae FG |
886 | return this; |
887 | } | |
888 | ||
889 | /** | |
890 | * Controls the number of shards for the block compressed cache. | |
891 | * This is applied only if blockCompressedCacheSize is set to non-negative. | |
892 | * | |
893 | * @return numShardBits the number of shard bits. The resulting | |
894 | * number of shards would be 2 ^ numShardBits. Any negative | |
895 | * number means use default settings. | |
896 | */ | |
494da23a | 897 | @Deprecated |
7c673cae | 898 | public int blockCacheCompressedNumShardBits() { |
494da23a | 899 | return blockCacheCompressedNumShardBits; |
7c673cae FG |
900 | } |
901 | ||
902 | /** | |
903 | * Controls the number of shards for the block compressed cache. | |
904 | * This is applied only if blockCompressedCacheSize is set to non-negative. | |
905 | * | |
906 | * @param blockCacheCompressedNumShardBits the number of shard bits. The resulting | |
907 | * number of shards would be 2 ^ numShardBits. Any negative | |
908 | * number means use default settings." | |
909 | * @return the reference to the current option. | |
494da23a TL |
910 | * |
911 | * @deprecated Use {@link #setBlockCacheCompressed(Cache)}. | |
7c673cae | 912 | */ |
494da23a | 913 | @Deprecated |
7c673cae FG |
914 | public BlockBasedTableConfig setBlockCacheCompressedNumShardBits( |
915 | final int blockCacheCompressedNumShardBits) { | |
494da23a | 916 | this.blockCacheCompressedNumShardBits = blockCacheCompressedNumShardBits; |
7c673cae FG |
917 | return this; |
918 | } | |
919 | ||
920 | /** | |
494da23a TL |
921 | * Influence the behavior when kHashSearch is used. |
922 | * if false, stores a precise prefix to block range mapping | |
923 | * if true, does not store prefix and allows prefix hash collision | |
924 | * (less memory consumption) | |
7c673cae | 925 | * |
494da23a | 926 | * @return if hash collisions should be allowed. |
7c673cae | 927 | * |
494da23a TL |
928 | * @deprecated This option is now deprecated. No matter what value it |
929 | * is set to, it will behave as | |
930 | * if {@link #hashIndexAllowCollision()} == true. | |
7c673cae | 931 | */ |
494da23a TL |
932 | @Deprecated |
933 | public boolean hashIndexAllowCollision() { | |
934 | return true; | |
7c673cae FG |
935 | } |
936 | ||
937 | /** | |
494da23a TL |
938 | * Influence the behavior when kHashSearch is used. |
939 | * if false, stores a precise prefix to block range mapping | |
940 | * if true, does not store prefix and allows prefix hash collision | |
941 | * (less memory consumption) | |
7c673cae | 942 | * |
494da23a | 943 | * @param hashIndexAllowCollision points out if hash collisions should be allowed. |
7c673cae | 944 | * |
494da23a | 945 | * @return the reference to the current config. |
7c673cae | 946 | * |
494da23a TL |
947 | * @deprecated This option is now deprecated. No matter what value it |
948 | * is set to, it will behave as | |
949 | * if {@link #hashIndexAllowCollision()} == true. | |
7c673cae | 950 | */ |
494da23a TL |
951 | @Deprecated |
952 | public BlockBasedTableConfig setHashIndexAllowCollision( | |
953 | final boolean hashIndexAllowCollision) { | |
954 | // no-op | |
7c673cae FG |
955 | return this; |
956 | } | |
957 | ||
494da23a TL |
958 | @Override protected long newTableFactoryHandle() { |
959 | final long filterPolicyHandle; | |
960 | if (filterPolicy != null) { | |
961 | filterPolicyHandle = filterPolicy.nativeHandle_; | |
962 | } else { | |
963 | filterPolicyHandle = 0; | |
964 | } | |
7c673cae | 965 | |
494da23a TL |
966 | final long blockCacheHandle; |
967 | if (blockCache != null) { | |
968 | blockCacheHandle = blockCache.nativeHandle_; | |
969 | } else { | |
970 | blockCacheHandle = 0; | |
971 | } | |
7c673cae | 972 | |
494da23a TL |
973 | final long persistentCacheHandle; |
974 | if (persistentCache != null) { | |
975 | persistentCacheHandle = persistentCache.nativeHandle_; | |
976 | } else { | |
977 | persistentCacheHandle = 0; | |
7c673cae FG |
978 | } |
979 | ||
494da23a TL |
980 | final long blockCacheCompressedHandle; |
981 | if (blockCacheCompressed != null) { | |
982 | blockCacheCompressedHandle = blockCacheCompressed.nativeHandle_; | |
983 | } else { | |
984 | blockCacheCompressedHandle = 0; | |
11fdf7f2 TL |
985 | } |
986 | ||
494da23a | 987 | return newTableFactoryHandle(cacheIndexAndFilterBlocks, |
20effc67 TL |
988 | cacheIndexAndFilterBlocksWithHighPriority, pinL0FilterAndIndexBlocksInCache, |
989 | pinTopLevelIndexAndFilter, indexType.getValue(), dataBlockIndexType.getValue(), | |
990 | dataBlockHashTableUtilRatio, checksumType.getValue(), noBlockCache, blockCacheHandle, | |
991 | persistentCacheHandle, blockCacheCompressedHandle, blockSize, blockSizeDeviation, | |
992 | blockRestartInterval, indexBlockRestartInterval, metadataBlockSize, partitionFilters, | |
993 | optimizeFiltersForMemory, useDeltaEncoding, filterPolicyHandle, wholeKeyFiltering, | |
994 | verifyCompression, readAmpBytesPerBit, formatVersion, enableIndexCompression, blockAlign, | |
995 | indexShortening.getValue(), blockCacheSize, blockCacheNumShardBits, | |
494da23a TL |
996 | blockCacheCompressedSize, blockCacheCompressedNumShardBits); |
997 | } | |
998 | ||
20effc67 | 999 | private native long newTableFactoryHandle(final boolean cacheIndexAndFilterBlocks, |
494da23a | 1000 | final boolean cacheIndexAndFilterBlocksWithHighPriority, |
20effc67 TL |
1001 | final boolean pinL0FilterAndIndexBlocksInCache, final boolean pinTopLevelIndexAndFilter, |
1002 | final byte indexTypeValue, final byte dataBlockIndexTypeValue, | |
1003 | final double dataBlockHashTableUtilRatio, final byte checksumTypeValue, | |
1004 | final boolean noBlockCache, final long blockCacheHandle, final long persistentCacheHandle, | |
1005 | final long blockCacheCompressedHandle, final long blockSize, final int blockSizeDeviation, | |
1006 | final int blockRestartInterval, final int indexBlockRestartInterval, | |
1007 | final long metadataBlockSize, final boolean partitionFilters, | |
1008 | final boolean optimizeFiltersForMemory, final boolean useDeltaEncoding, | |
1009 | final long filterPolicyHandle, final boolean wholeKeyFiltering, | |
1010 | final boolean verifyCompression, final int readAmpBytesPerBit, final int formatVersion, | |
1011 | final boolean enableIndexCompression, final boolean blockAlign, final byte indexShortening, | |
1012 | ||
1013 | @Deprecated final long blockCacheSize, @Deprecated final int blockCacheNumShardBits, | |
494da23a TL |
1014 | |
1015 | @Deprecated final long blockCacheCompressedSize, | |
20effc67 | 1016 | @Deprecated final int blockCacheCompressedNumShardBits); |
494da23a TL |
1017 | |
1018 | //TODO(AR) flushBlockPolicyFactory | |
1019 | private boolean cacheIndexAndFilterBlocks; | |
1020 | private boolean cacheIndexAndFilterBlocksWithHighPriority; | |
1021 | private boolean pinL0FilterAndIndexBlocksInCache; | |
1022 | private boolean pinTopLevelIndexAndFilter; | |
1023 | private IndexType indexType; | |
1024 | private DataBlockIndexType dataBlockIndexType; | |
1025 | private double dataBlockHashTableUtilRatio; | |
1026 | private ChecksumType checksumType; | |
1027 | private boolean noBlockCache; | |
1028 | private Cache blockCache; | |
1029 | private PersistentCache persistentCache; | |
1030 | private Cache blockCacheCompressed; | |
1031 | private long blockSize; | |
1032 | private int blockSizeDeviation; | |
1033 | private int blockRestartInterval; | |
1034 | private int indexBlockRestartInterval; | |
1035 | private long metadataBlockSize; | |
1036 | private boolean partitionFilters; | |
20effc67 | 1037 | private boolean optimizeFiltersForMemory; |
494da23a TL |
1038 | private boolean useDeltaEncoding; |
1039 | private Filter filterPolicy; | |
1040 | private boolean wholeKeyFiltering; | |
1041 | private boolean verifyCompression; | |
1042 | private int readAmpBytesPerBit; | |
1043 | private int formatVersion; | |
1044 | private boolean enableIndexCompression; | |
1045 | private boolean blockAlign; | |
20effc67 | 1046 | private IndexShorteningMode indexShortening; |
494da23a TL |
1047 | |
1048 | // NOTE: ONLY used if blockCache == null | |
1049 | @Deprecated private long blockCacheSize; | |
1050 | @Deprecated private int blockCacheNumShardBits; | |
1051 | ||
1052 | // NOTE: ONLY used if blockCacheCompressed == null | |
1053 | @Deprecated private long blockCacheCompressedSize; | |
1054 | @Deprecated private int blockCacheCompressedNumShardBits; | |
7c673cae | 1055 | } |