ceph/src/rocksdb/java/src/test/java/org/rocksdb/BlockBasedTableConfigTest.java

   1 // Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
   2 //  This source code is licensed under both the GPLv2 (found in the
   3 //  COPYING file in the root directory) and Apache 2.0 License
   4 //  (found in the LICENSE.Apache file in the root directory).
   5
   6 package org.rocksdb;
   7
   8 import org.junit.ClassRule;
   9 import org.junit.Ignore;
  10 import org.junit.Rule;
  11 import org.junit.Test;
  12 import org.junit.rules.TemporaryFolder;
  13
  14 import java.nio.charset.StandardCharsets;
  15
  16 import static org.assertj.core.api.Assertions.assertThat;
  17
  18 public class BlockBasedTableConfigTest {
  19
  20   @ClassRule
  21   public static final RocksMemoryResource rocksMemoryResource =
  22       new RocksMemoryResource();
  23
  24   @Rule public TemporaryFolder dbFolder = new TemporaryFolder();
  25
  26   @Test
  27   public void cacheIndexAndFilterBlocks() {
  28     final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
  29     blockBasedTableConfig.setCacheIndexAndFilterBlocks(true);
  30     assertThat(blockBasedTableConfig.cacheIndexAndFilterBlocks()).
  31         isTrue();
  32
  33   }
  34
  35   @Test
  36   public void cacheIndexAndFilterBlocksWithHighPriority() {
  37     final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
  38     blockBasedTableConfig.setCacheIndexAndFilterBlocksWithHighPriority(true);
  39     assertThat(blockBasedTableConfig.cacheIndexAndFilterBlocksWithHighPriority()).
  40         isTrue();
  41   }
  42
  43   @Test
  44   public void pinL0FilterAndIndexBlocksInCache() {
  45     final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
  46     blockBasedTableConfig.setPinL0FilterAndIndexBlocksInCache(true);
  47     assertThat(blockBasedTableConfig.pinL0FilterAndIndexBlocksInCache()).
  48         isTrue();
  49   }
  50
  51   @Test
  52   public void pinTopLevelIndexAndFilter() {
  53     final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
  54     blockBasedTableConfig.setPinTopLevelIndexAndFilter(false);
  55     assertThat(blockBasedTableConfig.pinTopLevelIndexAndFilter()).
  56         isFalse();
  57   }
  58
  59   @Test
  60   public void indexType() {
  61     final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
  62     assertThat(IndexType.values().length).isEqualTo(3);
  63     blockBasedTableConfig.setIndexType(IndexType.kHashSearch);
  64     assertThat(blockBasedTableConfig.indexType().equals(
  65         IndexType.kHashSearch));
  66     assertThat(IndexType.valueOf("kBinarySearch")).isNotNull();
  67     blockBasedTableConfig.setIndexType(IndexType.valueOf("kBinarySearch"));
  68     assertThat(blockBasedTableConfig.indexType().equals(
  69         IndexType.kBinarySearch));
  70   }
  71
  72   @Test
  73   public void dataBlockIndexType() {
  74     final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
  75     blockBasedTableConfig.setDataBlockIndexType(DataBlockIndexType.kDataBlockBinaryAndHash);
  76     assertThat(blockBasedTableConfig.dataBlockIndexType().equals(
  77         DataBlockIndexType.kDataBlockBinaryAndHash));
  78     blockBasedTableConfig.setDataBlockIndexType(DataBlockIndexType.kDataBlockBinarySearch);
  79     assertThat(blockBasedTableConfig.dataBlockIndexType().equals(
  80         DataBlockIndexType.kDataBlockBinarySearch));
  81   }
  82
  83   @Test
  84   public void checksumType() {
  85     final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
  86     assertThat(ChecksumType.values().length).isEqualTo(3);
  87     assertThat(ChecksumType.valueOf("kxxHash")).
  88         isEqualTo(ChecksumType.kxxHash);
  89     blockBasedTableConfig.setChecksumType(ChecksumType.kNoChecksum);
  90     blockBasedTableConfig.setChecksumType(ChecksumType.kxxHash);
  91     assertThat(blockBasedTableConfig.checksumType().equals(
  92         ChecksumType.kxxHash));
  93   }
  94
  95   @Test
  96   public void noBlockCache() {
  97     final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
  98     blockBasedTableConfig.setNoBlockCache(true);
  99     assertThat(blockBasedTableConfig.noBlockCache()).isTrue();
 100   }
 101
 102   @Test
 103   public void blockCache() {
 104     try (
 105         final Cache cache = new LRUCache(17 * 1024 * 1024);
 106         final Options options = new Options().setTableFormatConfig(
 107             new BlockBasedTableConfig().setBlockCache(cache))) {
 108       assertThat(options.tableFactoryName()).isEqualTo("BlockBasedTable");
 109     }
 110   }
 111
 112   @Test
 113   public void blockCacheIntegration() throws RocksDBException {
 114     try (final Cache cache = new LRUCache(8 * 1024 * 1024);
 115          final Statistics statistics = new Statistics()) {
 116       for (int shard = 0; shard < 8; shard++) {
 117         try (final Options options =
 118                  new Options()
 119                      .setCreateIfMissing(true)
 120                      .setStatistics(statistics)
 121                      .setTableFormatConfig(new BlockBasedTableConfig().setBlockCache(cache));
 122              final RocksDB db =
 123                  RocksDB.open(options, dbFolder.getRoot().getAbsolutePath() + "/" + shard)) {
 124           final byte[] key = "some-key".getBytes(StandardCharsets.UTF_8);
 125           final byte[] value = "some-value".getBytes(StandardCharsets.UTF_8);
 126
 127           db.put(key, value);
 128           db.flush(new FlushOptions());
 129           db.get(key);
 130
 131           assertThat(statistics.getTickerCount(TickerType.BLOCK_CACHE_ADD)).isEqualTo(shard + 1);
 132         }
 133       }
 134     }
 135   }
 136
 137   @Test
 138   public void persistentCache() throws RocksDBException {
 139     try (final DBOptions dbOptions = new DBOptions().
 140         setInfoLogLevel(InfoLogLevel.INFO_LEVEL).
 141         setCreateIfMissing(true);
 142         final Logger logger = new Logger(dbOptions) {
 143       @Override
 144       protected void log(final InfoLogLevel infoLogLevel, final String logMsg) {
 145         System.out.println(infoLogLevel.name() + ": " + logMsg);
 146       }
 147     }) {
 148       try (final PersistentCache persistentCache =
 149                new PersistentCache(Env.getDefault(), dbFolder.getRoot().getPath(), 1024 * 1024 * 100, logger, false);
 150            final Options options = new Options().setTableFormatConfig(
 151                new BlockBasedTableConfig().setPersistentCache(persistentCache))) {
 152         assertThat(options.tableFactoryName()).isEqualTo("BlockBasedTable");
 153       }
 154     }
 155   }
 156
 157   @Test
 158   public void blockCacheCompressed() {
 159     try (final Cache cache = new LRUCache(17 * 1024 * 1024);
 160          final Options options = new Options().setTableFormatConfig(
 161         new BlockBasedTableConfig().setBlockCacheCompressed(cache))) {
 162       assertThat(options.tableFactoryName()).isEqualTo("BlockBasedTable");
 163     }
 164   }
 165
 166   @Ignore("See issue: https://github.com/facebook/rocksdb/issues/4822")
 167   @Test
 168   public void blockCacheCompressedIntegration() throws RocksDBException {
 169     final byte[] key1 = "some-key1".getBytes(StandardCharsets.UTF_8);
 170     final byte[] key2 = "some-key1".getBytes(StandardCharsets.UTF_8);
 171     final byte[] key3 = "some-key1".getBytes(StandardCharsets.UTF_8);
 172     final byte[] key4 = "some-key1".getBytes(StandardCharsets.UTF_8);
 173     final byte[] value = "some-value".getBytes(StandardCharsets.UTF_8);
 174
 175     try (final Cache compressedCache = new LRUCache(8 * 1024 * 1024);
 176          final Statistics statistics = new Statistics()) {
 177
 178       final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig()
 179           .setNoBlockCache(true)
 180           .setBlockCache(null)
 181           .setBlockCacheCompressed(compressedCache)
 182           .setFormatVersion(4);
 183
 184       try (final Options options = new Options()
 185              .setCreateIfMissing(true)
 186              .setStatistics(statistics)
 187              .setTableFormatConfig(blockBasedTableConfig)) {
 188
 189         for (int shard = 0; shard < 8; shard++) {
 190           try (final FlushOptions flushOptions = new FlushOptions();
 191                final WriteOptions writeOptions = new WriteOptions();
 192                final ReadOptions readOptions = new ReadOptions();
 193                final RocksDB db =
 194                    RocksDB.open(options, dbFolder.getRoot().getAbsolutePath() + "/" + shard)) {
 195
 196             db.put(writeOptions, key1, value);
 197             db.put(writeOptions, key2, value);
 198             db.put(writeOptions, key3, value);
 199             db.put(writeOptions, key4, value);
 200             db.flush(flushOptions);
 201
 202             db.get(readOptions, key1);
 203             db.get(readOptions, key2);
 204             db.get(readOptions, key3);
 205             db.get(readOptions, key4);
 206
 207             assertThat(statistics.getTickerCount(TickerType.BLOCK_CACHE_COMPRESSED_ADD)).isEqualTo(shard + 1);
 208           }
 209         }
 210       }
 211     }
 212   }
 213
 214   @Test
 215   public void blockSize() {
 216     final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
 217     blockBasedTableConfig.setBlockSize(10);
 218     assertThat(blockBasedTableConfig.blockSize()).isEqualTo(10);
 219   }
 220
 221   @Test
 222   public void blockSizeDeviation() {
 223     final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
 224     blockBasedTableConfig.setBlockSizeDeviation(12);
 225     assertThat(blockBasedTableConfig.blockSizeDeviation()).
 226         isEqualTo(12);
 227   }
 228
 229   @Test
 230   public void blockRestartInterval() {
 231     final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
 232     blockBasedTableConfig.setBlockRestartInterval(15);
 233     assertThat(blockBasedTableConfig.blockRestartInterval()).
 234         isEqualTo(15);
 235   }
 236
 237   @Test
 238   public void indexBlockRestartInterval() {
 239     final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
 240     blockBasedTableConfig.setIndexBlockRestartInterval(15);
 241     assertThat(blockBasedTableConfig.indexBlockRestartInterval()).
 242         isEqualTo(15);
 243   }
 244
 245   @Test
 246   public void metadataBlockSize() {
 247     final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
 248     blockBasedTableConfig.setMetadataBlockSize(1024);
 249     assertThat(blockBasedTableConfig.metadataBlockSize()).
 250         isEqualTo(1024);
 251   }
 252
 253   @Test
 254   public void partitionFilters() {
 255     final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
 256     blockBasedTableConfig.setPartitionFilters(true);
 257     assertThat(blockBasedTableConfig.partitionFilters()).
 258         isTrue();
 259   }
 260
 261   @Test
 262   public void useDeltaEncoding() {
 263     final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
 264     blockBasedTableConfig.setUseDeltaEncoding(false);
 265     assertThat(blockBasedTableConfig.useDeltaEncoding()).
 266         isFalse();
 267   }
 268
 269   @Test
 270   public void blockBasedTableWithFilterPolicy() {
 271     try(final Options options = new Options()
 272         .setTableFormatConfig(new BlockBasedTableConfig()
 273             .setFilterPolicy(new BloomFilter(10)))) {
 274       assertThat(options.tableFactoryName()).
 275           isEqualTo("BlockBasedTable");
 276     }
 277   }
 278
 279   @Test
 280   public void blockBasedTableWithoutFilterPolicy() {
 281     try(final Options options = new Options().setTableFormatConfig(
 282         new BlockBasedTableConfig().setFilterPolicy(null))) {
 283       assertThat(options.tableFactoryName()).
 284           isEqualTo("BlockBasedTable");
 285     }
 286   }
 287
 288   @Test
 289   public void wholeKeyFiltering() {
 290     final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
 291     blockBasedTableConfig.setWholeKeyFiltering(false);
 292     assertThat(blockBasedTableConfig.wholeKeyFiltering()).
 293         isFalse();
 294   }
 295
 296   @Test
 297   public void verifyCompression() {
 298     final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
 299     blockBasedTableConfig.setVerifyCompression(true);
 300     assertThat(blockBasedTableConfig.verifyCompression()).
 301         isTrue();
 302   }
 303
 304   @Test
 305   public void readAmpBytesPerBit() {
 306     final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
 307     blockBasedTableConfig.setReadAmpBytesPerBit(2);
 308     assertThat(blockBasedTableConfig.readAmpBytesPerBit()).
 309         isEqualTo(2);
 310   }
 311
 312   @Test
 313   public void formatVersion() {
 314     final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
 315     for (int version = 0; version < 5; version++) {
 316       blockBasedTableConfig.setFormatVersion(version);
 317       assertThat(blockBasedTableConfig.formatVersion()).isEqualTo(version);
 318     }
 319   }
 320
 321   @Test(expected = AssertionError.class)
 322   public void formatVersionFailNegative() {
 323     final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
 324     blockBasedTableConfig.setFormatVersion(-1);
 325   }
 326
 327   @Test(expected = AssertionError.class)
 328   public void formatVersionFailIllegalVersion() {
 329     final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
 330     blockBasedTableConfig.setFormatVersion(99);
 331   }
 332
 333   @Test
 334   public void enableIndexCompression() {
 335     final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
 336     blockBasedTableConfig.setEnableIndexCompression(false);
 337     assertThat(blockBasedTableConfig.enableIndexCompression()).
 338         isFalse();
 339   }
 340
 341   @Test
 342   public void blockAlign() {
 343     final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
 344     blockBasedTableConfig.setBlockAlign(true);
 345     assertThat(blockBasedTableConfig.blockAlign()).
 346         isTrue();
 347   }
 348
 349   @Deprecated
 350   @Test
 351   public void hashIndexAllowCollision() {
 352     final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
 353     blockBasedTableConfig.setHashIndexAllowCollision(false);
 354     assertThat(blockBasedTableConfig.hashIndexAllowCollision()).
 355         isTrue();  // NOTE: setHashIndexAllowCollision should do nothing!
 356   }
 357
 358   @Deprecated
 359   @Test
 360   public void blockCacheSize() {
 361     final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
 362     blockBasedTableConfig.setBlockCacheSize(8 * 1024);
 363     assertThat(blockBasedTableConfig.blockCacheSize()).
 364         isEqualTo(8 * 1024);
 365   }
 366
 367   @Deprecated
 368   @Test
 369   public void blockCacheNumShardBits() {
 370     final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
 371     blockBasedTableConfig.setCacheNumShardBits(5);
 372     assertThat(blockBasedTableConfig.cacheNumShardBits()).
 373         isEqualTo(5);
 374   }
 375
 376   @Deprecated
 377   @Test
 378   public void blockCacheCompressedSize() {
 379     final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
 380     blockBasedTableConfig.setBlockCacheCompressedSize(40);
 381     assertThat(blockBasedTableConfig.blockCacheCompressedSize()).
 382         isEqualTo(40);
 383   }
 384
 385   @Deprecated
 386   @Test
 387   public void blockCacheCompressedNumShardBits() {
 388     final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
 389     blockBasedTableConfig.setBlockCacheCompressedNumShardBits(4);
 390     assertThat(blockBasedTableConfig.blockCacheCompressedNumShardBits()).
 391         isEqualTo(4);
 392   }
 393 }