ceph/src/rocksdb/java/src/test/java/org/rocksdb/BlockBasedTableConfigTest.java

   1 // Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
   2 //  This source code is licensed under both the GPLv2 (found in the
   3 //  COPYING file in the root directory) and Apache 2.0 License
   4 //  (found in the LICENSE.Apache file in the root directory).
   5
   6 package org.rocksdb;
   7
   8 import static org.assertj.core.api.Assertions.assertThat;
   9 import static org.junit.Assert.fail;
  10
  11 import java.nio.charset.StandardCharsets;
  12 import java.nio.file.Files;
  13 import java.nio.file.Path;
  14 import java.nio.file.Paths;
  15 import java.util.stream.Stream;
  16 import org.junit.ClassRule;
  17 import org.junit.Ignore;
  18 import org.junit.Rule;
  19 import org.junit.Test;
  20 import org.junit.rules.TemporaryFolder;
  21
  22 public class BlockBasedTableConfigTest {
  23
  24   @ClassRule
  25   public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE =
  26       new RocksNativeLibraryResource();
  27
  28   @Rule public TemporaryFolder dbFolder = new TemporaryFolder();
  29
  30   @Test
  31   public void cacheIndexAndFilterBlocks() {
  32     final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
  33     blockBasedTableConfig.setCacheIndexAndFilterBlocks(true);
  34     assertThat(blockBasedTableConfig.cacheIndexAndFilterBlocks()).
  35         isTrue();
  36   }
  37
  38   @Test
  39   public void cacheIndexAndFilterBlocksWithHighPriority() {
  40     final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
  41     assertThat(blockBasedTableConfig.cacheIndexAndFilterBlocksWithHighPriority()).
  42         isTrue();
  43     blockBasedTableConfig.setCacheIndexAndFilterBlocksWithHighPriority(false);
  44     assertThat(blockBasedTableConfig.cacheIndexAndFilterBlocksWithHighPriority()).isFalse();
  45   }
  46
  47   @Test
  48   public void pinL0FilterAndIndexBlocksInCache() {
  49     final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
  50     blockBasedTableConfig.setPinL0FilterAndIndexBlocksInCache(true);
  51     assertThat(blockBasedTableConfig.pinL0FilterAndIndexBlocksInCache()).
  52         isTrue();
  53   }
  54
  55   @Test
  56   public void pinTopLevelIndexAndFilter() {
  57     final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
  58     blockBasedTableConfig.setPinTopLevelIndexAndFilter(false);
  59     assertThat(blockBasedTableConfig.pinTopLevelIndexAndFilter()).
  60         isFalse();
  61   }
  62
  63   @Test
  64   public void indexType() {
  65     final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
  66     assertThat(IndexType.values().length).isEqualTo(4);
  67     blockBasedTableConfig.setIndexType(IndexType.kHashSearch);
  68     assertThat(blockBasedTableConfig.indexType()).isEqualTo(IndexType.kHashSearch);
  69     assertThat(IndexType.valueOf("kBinarySearch")).isNotNull();
  70     blockBasedTableConfig.setIndexType(IndexType.valueOf("kBinarySearch"));
  71     assertThat(blockBasedTableConfig.indexType()).isEqualTo(IndexType.kBinarySearch);
  72   }
  73
  74   @Test
  75   public void dataBlockIndexType() {
  76     final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
  77     blockBasedTableConfig.setDataBlockIndexType(DataBlockIndexType.kDataBlockBinaryAndHash);
  78     assertThat(blockBasedTableConfig.dataBlockIndexType())
  79         .isEqualTo(DataBlockIndexType.kDataBlockBinaryAndHash);
  80     blockBasedTableConfig.setDataBlockIndexType(DataBlockIndexType.kDataBlockBinarySearch);
  81     assertThat(blockBasedTableConfig.dataBlockIndexType())
  82         .isEqualTo(DataBlockIndexType.kDataBlockBinarySearch);
  83   }
  84
  85   @Test
  86   public void checksumType() {
  87     final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
  88     assertThat(ChecksumType.values().length).isEqualTo(5);
  89     assertThat(ChecksumType.valueOf("kxxHash")).
  90         isEqualTo(ChecksumType.kxxHash);
  91     blockBasedTableConfig.setChecksumType(ChecksumType.kNoChecksum);
  92     assertThat(blockBasedTableConfig.checksumType()).isEqualTo(ChecksumType.kNoChecksum);
  93     blockBasedTableConfig.setChecksumType(ChecksumType.kxxHash);
  94     assertThat(blockBasedTableConfig.checksumType()).isEqualTo(ChecksumType.kxxHash);
  95     blockBasedTableConfig.setChecksumType(ChecksumType.kxxHash64);
  96     assertThat(blockBasedTableConfig.checksumType()).isEqualTo(ChecksumType.kxxHash64);
  97     blockBasedTableConfig.setChecksumType(ChecksumType.kXXH3);
  98     assertThat(blockBasedTableConfig.checksumType()).isEqualTo(ChecksumType.kXXH3);
  99   }
 100
 101   @Test
 102   public void jniPortal() throws Exception {
 103     // Verifies that the JNI layer is correctly translating options.
 104     // Since introspecting the options requires creating a database, the checks
 105     // cover multiple options at the same time.
 106
 107     final BlockBasedTableConfig tableConfig = new BlockBasedTableConfig();
 108
 109     tableConfig.setIndexType(IndexType.kBinarySearch);
 110     tableConfig.setDataBlockIndexType(DataBlockIndexType.kDataBlockBinarySearch);
 111     tableConfig.setChecksumType(ChecksumType.kNoChecksum);
 112     try (final Options options = new Options().setTableFormatConfig(tableConfig)) {
 113       String opts = getOptionAsString(options);
 114       assertThat(opts).contains("index_type=kBinarySearch");
 115       assertThat(opts).contains("data_block_index_type=kDataBlockBinarySearch");
 116       assertThat(opts).contains("checksum=kNoChecksum");
 117     }
 118
 119     tableConfig.setIndexType(IndexType.kHashSearch);
 120     tableConfig.setDataBlockIndexType(DataBlockIndexType.kDataBlockBinaryAndHash);
 121     tableConfig.setChecksumType(ChecksumType.kCRC32c);
 122     try (final Options options = new Options().setTableFormatConfig(tableConfig)) {
 123       options.useCappedPrefixExtractor(1); // Needed to use kHashSearch
 124       String opts = getOptionAsString(options);
 125       assertThat(opts).contains("index_type=kHashSearch");
 126       assertThat(opts).contains("data_block_index_type=kDataBlockBinaryAndHash");
 127       assertThat(opts).contains("checksum=kCRC32c");
 128     }
 129
 130     tableConfig.setIndexType(IndexType.kTwoLevelIndexSearch);
 131     tableConfig.setChecksumType(ChecksumType.kxxHash);
 132     try (final Options options = new Options().setTableFormatConfig(tableConfig)) {
 133       String opts = getOptionAsString(options);
 134       assertThat(opts).contains("index_type=kTwoLevelIndexSearch");
 135       assertThat(opts).contains("checksum=kxxHash");
 136     }
 137
 138     tableConfig.setIndexType(IndexType.kBinarySearchWithFirstKey);
 139     tableConfig.setChecksumType(ChecksumType.kxxHash64);
 140     try (final Options options = new Options().setTableFormatConfig(tableConfig)) {
 141       String opts = getOptionAsString(options);
 142       assertThat(opts).contains("index_type=kBinarySearchWithFirstKey");
 143       assertThat(opts).contains("checksum=kxxHash64");
 144     }
 145
 146     tableConfig.setChecksumType(ChecksumType.kXXH3);
 147     try (final Options options = new Options().setTableFormatConfig(tableConfig)) {
 148       String opts = getOptionAsString(options);
 149       assertThat(opts).contains("checksum=kXXH3");
 150     }
 151   }
 152
 153   private String getOptionAsString(Options options) throws Exception {
 154     options.setCreateIfMissing(true);
 155     String dbPath = dbFolder.getRoot().getAbsolutePath();
 156     String result;
 157     try (final RocksDB db = RocksDB.open(options, dbPath);
 158          final Stream<Path> pathStream = Files.walk(Paths.get(dbPath))) {
 159       Path optionsPath =
 160           pathStream
 161               .filter(p -> p.getFileName().toString().startsWith("OPTIONS"))
 162               .findAny()
 163               .orElseThrow(() -> new AssertionError("Missing options file"));
 164       byte[] optionsData = Files.readAllBytes(optionsPath);
 165       result = new String(optionsData, StandardCharsets.UTF_8);
 166     }
 167     RocksDB.destroyDB(dbPath, options);
 168     return result;
 169   }
 170
 171   @Test
 172   public void noBlockCache() {
 173     final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
 174     blockBasedTableConfig.setNoBlockCache(true);
 175     assertThat(blockBasedTableConfig.noBlockCache()).isTrue();
 176   }
 177
 178   @Test
 179   public void blockCache() {
 180     try (
 181         final Cache cache = new LRUCache(17 * 1024 * 1024);
 182         final Options options = new Options().setTableFormatConfig(
 183             new BlockBasedTableConfig().setBlockCache(cache))) {
 184       assertThat(options.tableFactoryName()).isEqualTo("BlockBasedTable");
 185     }
 186   }
 187
 188   @Test
 189   public void blockCacheIntegration() throws RocksDBException {
 190     try (final Cache cache = new LRUCache(8 * 1024 * 1024);
 191          final Statistics statistics = new Statistics()) {
 192       for (int shard = 0; shard < 8; shard++) {
 193         try (final Options options =
 194                  new Options()
 195                      .setCreateIfMissing(true)
 196                      .setStatistics(statistics)
 197                      .setTableFormatConfig(new BlockBasedTableConfig().setBlockCache(cache));
 198              final RocksDB db =
 199                  RocksDB.open(options, dbFolder.getRoot().getAbsolutePath() + "/" + shard)) {
 200           final byte[] key = "some-key".getBytes(StandardCharsets.UTF_8);
 201           final byte[] value = "some-value".getBytes(StandardCharsets.UTF_8);
 202
 203           db.put(key, value);
 204           db.flush(new FlushOptions());
 205           db.get(key);
 206
 207           assertThat(statistics.getTickerCount(TickerType.BLOCK_CACHE_ADD)).isEqualTo(shard + 1);
 208         }
 209       }
 210     }
 211   }
 212
 213   @Test
 214   public void persistentCache() throws RocksDBException {
 215     try (final DBOptions dbOptions = new DBOptions().
 216         setInfoLogLevel(InfoLogLevel.INFO_LEVEL).
 217         setCreateIfMissing(true);
 218         final Logger logger = new Logger(dbOptions) {
 219       @Override
 220       protected void log(final InfoLogLevel infoLogLevel, final String logMsg) {
 221         System.out.println(infoLogLevel.name() + ": " + logMsg);
 222       }
 223     }) {
 224       try (final PersistentCache persistentCache =
 225                new PersistentCache(Env.getDefault(), dbFolder.getRoot().getPath(), 1024 * 1024 * 100, logger, false);
 226            final Options options = new Options().setTableFormatConfig(
 227                new BlockBasedTableConfig().setPersistentCache(persistentCache))) {
 228         assertThat(options.tableFactoryName()).isEqualTo("BlockBasedTable");
 229       }
 230     }
 231   }
 232
 233   @Test
 234   public void blockCacheCompressed() {
 235     try (final Cache cache = new LRUCache(17 * 1024 * 1024);
 236          final Options options = new Options().setTableFormatConfig(
 237         new BlockBasedTableConfig().setBlockCacheCompressed(cache))) {
 238       assertThat(options.tableFactoryName()).isEqualTo("BlockBasedTable");
 239     }
 240   }
 241
 242   @Ignore("See issue: https://github.com/facebook/rocksdb/issues/4822")
 243   @Test
 244   public void blockCacheCompressedIntegration() throws RocksDBException {
 245     final byte[] key1 = "some-key1".getBytes(StandardCharsets.UTF_8);
 246     final byte[] key2 = "some-key1".getBytes(StandardCharsets.UTF_8);
 247     final byte[] key3 = "some-key1".getBytes(StandardCharsets.UTF_8);
 248     final byte[] key4 = "some-key1".getBytes(StandardCharsets.UTF_8);
 249     final byte[] value = "some-value".getBytes(StandardCharsets.UTF_8);
 250
 251     try (final Cache compressedCache = new LRUCache(8 * 1024 * 1024);
 252          final Statistics statistics = new Statistics()) {
 253
 254       final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig()
 255           .setNoBlockCache(true)
 256           .setBlockCache(null)
 257           .setBlockCacheCompressed(compressedCache)
 258           .setFormatVersion(4);
 259
 260       try (final Options options = new Options()
 261              .setCreateIfMissing(true)
 262              .setStatistics(statistics)
 263              .setTableFormatConfig(blockBasedTableConfig)) {
 264
 265         for (int shard = 0; shard < 8; shard++) {
 266           try (final FlushOptions flushOptions = new FlushOptions();
 267                final WriteOptions writeOptions = new WriteOptions();
 268                final ReadOptions readOptions = new ReadOptions();
 269                final RocksDB db =
 270                    RocksDB.open(options, dbFolder.getRoot().getAbsolutePath() + "/" + shard)) {
 271
 272             db.put(writeOptions, key1, value);
 273             db.put(writeOptions, key2, value);
 274             db.put(writeOptions, key3, value);
 275             db.put(writeOptions, key4, value);
 276             db.flush(flushOptions);
 277
 278             db.get(readOptions, key1);
 279             db.get(readOptions, key2);
 280             db.get(readOptions, key3);
 281             db.get(readOptions, key4);
 282
 283             assertThat(statistics.getTickerCount(TickerType.BLOCK_CACHE_COMPRESSED_ADD)).isEqualTo(shard + 1);
 284           }
 285         }
 286       }
 287     }
 288   }
 289
 290   @Test
 291   public void blockSize() {
 292     final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
 293     blockBasedTableConfig.setBlockSize(10);
 294     assertThat(blockBasedTableConfig.blockSize()).isEqualTo(10);
 295   }
 296
 297   @Test
 298   public void blockSizeDeviation() {
 299     final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
 300     blockBasedTableConfig.setBlockSizeDeviation(12);
 301     assertThat(blockBasedTableConfig.blockSizeDeviation()).
 302         isEqualTo(12);
 303   }
 304
 305   @Test
 306   public void blockRestartInterval() {
 307     final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
 308     blockBasedTableConfig.setBlockRestartInterval(15);
 309     assertThat(blockBasedTableConfig.blockRestartInterval()).
 310         isEqualTo(15);
 311   }
 312
 313   @Test
 314   public void indexBlockRestartInterval() {
 315     final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
 316     blockBasedTableConfig.setIndexBlockRestartInterval(15);
 317     assertThat(blockBasedTableConfig.indexBlockRestartInterval()).
 318         isEqualTo(15);
 319   }
 320
 321   @Test
 322   public void metadataBlockSize() {
 323     final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
 324     blockBasedTableConfig.setMetadataBlockSize(1024);
 325     assertThat(blockBasedTableConfig.metadataBlockSize()).
 326         isEqualTo(1024);
 327   }
 328
 329   @Test
 330   public void partitionFilters() {
 331     final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
 332     blockBasedTableConfig.setPartitionFilters(true);
 333     assertThat(blockBasedTableConfig.partitionFilters()).
 334         isTrue();
 335   }
 336
 337   @Test
 338   public void optimizeFiltersForMemory() {
 339     final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
 340     blockBasedTableConfig.setOptimizeFiltersForMemory(true);
 341     assertThat(blockBasedTableConfig.optimizeFiltersForMemory()).isTrue();
 342   }
 343
 344   @Test
 345   public void useDeltaEncoding() {
 346     final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
 347     blockBasedTableConfig.setUseDeltaEncoding(false);
 348     assertThat(blockBasedTableConfig.useDeltaEncoding()).
 349         isFalse();
 350   }
 351
 352   @Test
 353   public void blockBasedTableWithFilterPolicy() {
 354     try(final Options options = new Options()
 355         .setTableFormatConfig(new BlockBasedTableConfig()
 356             .setFilterPolicy(new BloomFilter(10)))) {
 357       assertThat(options.tableFactoryName()).
 358           isEqualTo("BlockBasedTable");
 359     }
 360   }
 361
 362   @Test
 363   public void blockBasedTableWithoutFilterPolicy() {
 364     try(final Options options = new Options().setTableFormatConfig(
 365         new BlockBasedTableConfig().setFilterPolicy(null))) {
 366       assertThat(options.tableFactoryName()).
 367           isEqualTo("BlockBasedTable");
 368     }
 369   }
 370
 371   @Test
 372   public void wholeKeyFiltering() {
 373     final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
 374     blockBasedTableConfig.setWholeKeyFiltering(false);
 375     assertThat(blockBasedTableConfig.wholeKeyFiltering()).
 376         isFalse();
 377   }
 378
 379   @Test
 380   public void verifyCompression() {
 381     final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
 382     assertThat(blockBasedTableConfig.verifyCompression()).isFalse();
 383     blockBasedTableConfig.setVerifyCompression(true);
 384     assertThat(blockBasedTableConfig.verifyCompression()).
 385         isTrue();
 386   }
 387
 388   @Test
 389   public void readAmpBytesPerBit() {
 390     final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
 391     blockBasedTableConfig.setReadAmpBytesPerBit(2);
 392     assertThat(blockBasedTableConfig.readAmpBytesPerBit()).
 393         isEqualTo(2);
 394   }
 395
 396   @Test
 397   public void formatVersion() {
 398     final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
 399     for (int version = 0; version <= 5; version++) {
 400       blockBasedTableConfig.setFormatVersion(version);
 401       assertThat(blockBasedTableConfig.formatVersion()).isEqualTo(version);
 402     }
 403   }
 404
 405   @Test(expected = AssertionError.class)
 406   public void formatVersionFailNegative() {
 407     final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
 408     blockBasedTableConfig.setFormatVersion(-1);
 409   }
 410
 411   @Test(expected = RocksDBException.class)
 412   public void invalidFormatVersion() throws RocksDBException {
 413     final BlockBasedTableConfig blockBasedTableConfig =
 414         new BlockBasedTableConfig().setFormatVersion(99999);
 415
 416     try (final Options options = new Options().setTableFormatConfig(blockBasedTableConfig);
 417          final RocksDB db = RocksDB.open(options, dbFolder.getRoot().getAbsolutePath())) {
 418       fail("Opening the database with an invalid format_version should have raised an exception");
 419     }
 420   }
 421
 422   @Test
 423   public void enableIndexCompression() {
 424     final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
 425     blockBasedTableConfig.setEnableIndexCompression(false);
 426     assertThat(blockBasedTableConfig.enableIndexCompression()).
 427         isFalse();
 428   }
 429
 430   @Test
 431   public void blockAlign() {
 432     final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
 433     blockBasedTableConfig.setBlockAlign(true);
 434     assertThat(blockBasedTableConfig.blockAlign()).
 435         isTrue();
 436   }
 437
 438   @Test
 439   public void indexShortening() {
 440     final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
 441     blockBasedTableConfig.setIndexShortening(IndexShorteningMode.kShortenSeparatorsAndSuccessor);
 442     assertThat(blockBasedTableConfig.indexShortening())
 443         .isEqualTo(IndexShorteningMode.kShortenSeparatorsAndSuccessor);
 444   }
 445
 446   @Deprecated
 447   @Test
 448   public void hashIndexAllowCollision() {
 449     final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
 450     blockBasedTableConfig.setHashIndexAllowCollision(false);
 451     assertThat(blockBasedTableConfig.hashIndexAllowCollision()).
 452         isTrue();  // NOTE: setHashIndexAllowCollision should do nothing!
 453   }
 454
 455   @Deprecated
 456   @Test
 457   public void blockCacheSize() {
 458     final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
 459     blockBasedTableConfig.setBlockCacheSize(8 * 1024);
 460     assertThat(blockBasedTableConfig.blockCacheSize()).
 461         isEqualTo(8 * 1024);
 462   }
 463
 464   @Deprecated
 465   @Test
 466   public void blockCacheNumShardBits() {
 467     final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
 468     blockBasedTableConfig.setCacheNumShardBits(5);
 469     assertThat(blockBasedTableConfig.cacheNumShardBits()).
 470         isEqualTo(5);
 471   }
 472
 473   @Deprecated
 474   @Test
 475   public void blockCacheCompressedSize() {
 476     final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
 477     blockBasedTableConfig.setBlockCacheCompressedSize(40);
 478     assertThat(blockBasedTableConfig.blockCacheCompressedSize()).
 479         isEqualTo(40);
 480   }
 481
 482   @Deprecated
 483   @Test
 484   public void blockCacheCompressedNumShardBits() {
 485     final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
 486     blockBasedTableConfig.setBlockCacheCompressedNumShardBits(4);
 487     assertThat(blockBasedTableConfig.blockCacheCompressedNumShardBits()).
 488         isEqualTo(4);
 489   }
 490 }