ceph/src/rocksdb/java/src/test/java/org/rocksdb/BlobOptionsTest.java

   1 // Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
   2 //  This source code is licensed under both the GPLv2 (found in the
   3 //  COPYING file in the root directory) and Apache 2.0 License
   4 //  (found in the LICENSE.Apache file in the root directory).
   5 package org.rocksdb;
   6
   7 import static java.nio.charset.StandardCharsets.UTF_8;
   8 import static org.assertj.core.api.Assertions.assertThat;
   9
  10 import java.io.File;
  11 import java.io.FilenameFilter;
  12 import java.util.*;
  13 import org.junit.ClassRule;
  14 import org.junit.Rule;
  15 import org.junit.Test;
  16 import org.junit.rules.TemporaryFolder;
  17
  18 public class BlobOptionsTest {
  19   @ClassRule
  20   public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE =
  21       new RocksNativeLibraryResource();
  22
  23   @Rule public TemporaryFolder dbFolder = new TemporaryFolder();
  24
  25   final int minBlobSize = 65536;
  26   final int largeBlobSize = 65536 * 2;
  27
  28   /**
  29    * Count the files in the temporary folder which end with a particular suffix
  30    * Used to query the state of a test database to check if it is as the test expects
  31    *
  32    * @param endsWith the suffix to match
  33    * @return the number of files with a matching suffix
  34    */
  35   @SuppressWarnings("CallToStringConcatCanBeReplacedByOperator")
  36   private int countDBFiles(final String endsWith) {
  37     return Objects
  38         .requireNonNull(dbFolder.getRoot().list(new FilenameFilter() {
  39           @Override
  40           public boolean accept(File dir, String name) {
  41             return name.endsWith(endsWith);
  42           }
  43         }))
  44         .length;
  45   }
  46
  47   @SuppressWarnings("SameParameterValue")
  48   private byte[] small_key(String suffix) {
  49     return ("small_key_" + suffix).getBytes(UTF_8);
  50   }
  51
  52   @SuppressWarnings("SameParameterValue")
  53   private byte[] small_value(String suffix) {
  54     return ("small_value_" + suffix).getBytes(UTF_8);
  55   }
  56
  57   private byte[] large_key(String suffix) {
  58     return ("large_key_" + suffix).getBytes(UTF_8);
  59   }
  60
  61   private byte[] large_value(String repeat) {
  62     final byte[] large_value = ("" + repeat + "_" + largeBlobSize + "b").getBytes(UTF_8);
  63     final byte[] large_buffer = new byte[largeBlobSize];
  64     for (int pos = 0; pos < largeBlobSize; pos += large_value.length) {
  65       int numBytes = Math.min(large_value.length, large_buffer.length - pos);
  66       System.arraycopy(large_value, 0, large_buffer, pos, numBytes);
  67     }
  68     return large_buffer;
  69   }
  70
  71   @Test
  72   public void blobOptions() {
  73     try (final Options options = new Options()) {
  74       assertThat(options.enableBlobFiles()).isEqualTo(false);
  75       assertThat(options.minBlobSize()).isEqualTo(0);
  76       assertThat(options.blobCompressionType()).isEqualTo(CompressionType.NO_COMPRESSION);
  77       assertThat(options.enableBlobGarbageCollection()).isEqualTo(false);
  78       assertThat(options.blobFileSize()).isEqualTo(268435456L);
  79       assertThat(options.blobGarbageCollectionAgeCutoff()).isEqualTo(0.25);
  80       assertThat(options.blobGarbageCollectionForceThreshold()).isEqualTo(1.0);
  81       assertThat(options.blobCompactionReadaheadSize()).isEqualTo(0);
  82       assertThat(options.prepopulateBlobCache())
  83           .isEqualTo(PrepopulateBlobCache.PREPOPULATE_BLOB_DISABLE);
  84
  85       assertThat(options.setEnableBlobFiles(true)).isEqualTo(options);
  86       assertThat(options.setMinBlobSize(132768L)).isEqualTo(options);
  87       assertThat(options.setBlobCompressionType(CompressionType.BZLIB2_COMPRESSION))
  88           .isEqualTo(options);
  89       assertThat(options.setEnableBlobGarbageCollection(true)).isEqualTo(options);
  90       assertThat(options.setBlobFileSize(132768L)).isEqualTo(options);
  91       assertThat(options.setBlobGarbageCollectionAgeCutoff(0.89)).isEqualTo(options);
  92       assertThat(options.setBlobGarbageCollectionForceThreshold(0.80)).isEqualTo(options);
  93       assertThat(options.setBlobCompactionReadaheadSize(262144L)).isEqualTo(options);
  94       assertThat(options.setBlobFileStartingLevel(0)).isEqualTo(options);
  95       assertThat(options.setPrepopulateBlobCache(PrepopulateBlobCache.PREPOPULATE_BLOB_FLUSH_ONLY))
  96           .isEqualTo(options);
  97
  98       assertThat(options.enableBlobFiles()).isEqualTo(true);
  99       assertThat(options.minBlobSize()).isEqualTo(132768L);
 100       assertThat(options.blobCompressionType()).isEqualTo(CompressionType.BZLIB2_COMPRESSION);
 101       assertThat(options.enableBlobGarbageCollection()).isEqualTo(true);
 102       assertThat(options.blobFileSize()).isEqualTo(132768L);
 103       assertThat(options.blobGarbageCollectionAgeCutoff()).isEqualTo(0.89);
 104       assertThat(options.blobGarbageCollectionForceThreshold()).isEqualTo(0.80);
 105       assertThat(options.blobCompactionReadaheadSize()).isEqualTo(262144L);
 106       assertThat(options.blobFileStartingLevel()).isEqualTo(0);
 107       assertThat(options.prepopulateBlobCache())
 108           .isEqualTo(PrepopulateBlobCache.PREPOPULATE_BLOB_FLUSH_ONLY);
 109     }
 110   }
 111
 112   @Test
 113   public void blobColumnFamilyOptions() {
 114     try (final ColumnFamilyOptions columnFamilyOptions = new ColumnFamilyOptions()) {
 115       assertThat(columnFamilyOptions.enableBlobFiles()).isEqualTo(false);
 116       assertThat(columnFamilyOptions.minBlobSize()).isEqualTo(0);
 117       assertThat(columnFamilyOptions.blobCompressionType())
 118           .isEqualTo(CompressionType.NO_COMPRESSION);
 119       assertThat(columnFamilyOptions.enableBlobGarbageCollection()).isEqualTo(false);
 120       assertThat(columnFamilyOptions.blobFileSize()).isEqualTo(268435456L);
 121       assertThat(columnFamilyOptions.blobGarbageCollectionAgeCutoff()).isEqualTo(0.25);
 122       assertThat(columnFamilyOptions.blobGarbageCollectionForceThreshold()).isEqualTo(1.0);
 123       assertThat(columnFamilyOptions.blobCompactionReadaheadSize()).isEqualTo(0);
 124
 125       assertThat(columnFamilyOptions.setEnableBlobFiles(true)).isEqualTo(columnFamilyOptions);
 126       assertThat(columnFamilyOptions.setMinBlobSize(132768L)).isEqualTo(columnFamilyOptions);
 127       assertThat(columnFamilyOptions.setBlobCompressionType(CompressionType.BZLIB2_COMPRESSION))
 128           .isEqualTo(columnFamilyOptions);
 129       assertThat(columnFamilyOptions.setEnableBlobGarbageCollection(true))
 130           .isEqualTo(columnFamilyOptions);
 131       assertThat(columnFamilyOptions.setBlobFileSize(132768L)).isEqualTo(columnFamilyOptions);
 132       assertThat(columnFamilyOptions.setBlobGarbageCollectionAgeCutoff(0.89))
 133           .isEqualTo(columnFamilyOptions);
 134       assertThat(columnFamilyOptions.setBlobGarbageCollectionForceThreshold(0.80))
 135           .isEqualTo(columnFamilyOptions);
 136       assertThat(columnFamilyOptions.setBlobCompactionReadaheadSize(262144L))
 137           .isEqualTo(columnFamilyOptions);
 138       assertThat(columnFamilyOptions.setBlobFileStartingLevel(0)).isEqualTo(columnFamilyOptions);
 139       assertThat(columnFamilyOptions.setPrepopulateBlobCache(
 140                      PrepopulateBlobCache.PREPOPULATE_BLOB_DISABLE))
 141           .isEqualTo(columnFamilyOptions);
 142
 143       assertThat(columnFamilyOptions.enableBlobFiles()).isEqualTo(true);
 144       assertThat(columnFamilyOptions.minBlobSize()).isEqualTo(132768L);
 145       assertThat(columnFamilyOptions.blobCompressionType())
 146           .isEqualTo(CompressionType.BZLIB2_COMPRESSION);
 147       assertThat(columnFamilyOptions.enableBlobGarbageCollection()).isEqualTo(true);
 148       assertThat(columnFamilyOptions.blobFileSize()).isEqualTo(132768L);
 149       assertThat(columnFamilyOptions.blobGarbageCollectionAgeCutoff()).isEqualTo(0.89);
 150       assertThat(columnFamilyOptions.blobGarbageCollectionForceThreshold()).isEqualTo(0.80);
 151       assertThat(columnFamilyOptions.blobCompactionReadaheadSize()).isEqualTo(262144L);
 152       assertThat(columnFamilyOptions.blobFileStartingLevel()).isEqualTo(0);
 153       assertThat(columnFamilyOptions.prepopulateBlobCache())
 154           .isEqualTo(PrepopulateBlobCache.PREPOPULATE_BLOB_DISABLE);
 155     }
 156   }
 157
 158   @Test
 159   public void blobMutableColumnFamilyOptionsBuilder() {
 160     final MutableColumnFamilyOptions.MutableColumnFamilyOptionsBuilder builder =
 161         MutableColumnFamilyOptions.builder();
 162     builder.setEnableBlobFiles(true)
 163         .setMinBlobSize(1024)
 164         .setBlobFileSize(132768)
 165         .setBlobCompressionType(CompressionType.BZLIB2_COMPRESSION)
 166         .setEnableBlobGarbageCollection(true)
 167         .setBlobGarbageCollectionAgeCutoff(0.89)
 168         .setBlobGarbageCollectionForceThreshold(0.80)
 169         .setBlobCompactionReadaheadSize(262144)
 170         .setBlobFileStartingLevel(1)
 171         .setPrepopulateBlobCache(PrepopulateBlobCache.PREPOPULATE_BLOB_FLUSH_ONLY);
 172
 173     assertThat(builder.enableBlobFiles()).isEqualTo(true);
 174     assertThat(builder.minBlobSize()).isEqualTo(1024);
 175     assertThat(builder.blobFileSize()).isEqualTo(132768);
 176     assertThat(builder.blobCompressionType()).isEqualTo(CompressionType.BZLIB2_COMPRESSION);
 177     assertThat(builder.enableBlobGarbageCollection()).isEqualTo(true);
 178     assertThat(builder.blobGarbageCollectionAgeCutoff()).isEqualTo(0.89);
 179     assertThat(builder.blobGarbageCollectionForceThreshold()).isEqualTo(0.80);
 180     assertThat(builder.blobCompactionReadaheadSize()).isEqualTo(262144);
 181     assertThat(builder.blobFileStartingLevel()).isEqualTo(1);
 182     assertThat(builder.prepopulateBlobCache())
 183         .isEqualTo(PrepopulateBlobCache.PREPOPULATE_BLOB_FLUSH_ONLY);
 184
 185     builder.setEnableBlobFiles(false)
 186         .setMinBlobSize(4096)
 187         .setBlobFileSize(2048)
 188         .setBlobCompressionType(CompressionType.LZ4_COMPRESSION)
 189         .setEnableBlobGarbageCollection(false)
 190         .setBlobGarbageCollectionAgeCutoff(0.91)
 191         .setBlobGarbageCollectionForceThreshold(0.96)
 192         .setBlobCompactionReadaheadSize(1024)
 193         .setBlobFileStartingLevel(0)
 194         .setPrepopulateBlobCache(PrepopulateBlobCache.PREPOPULATE_BLOB_DISABLE);
 195
 196     assertThat(builder.enableBlobFiles()).isEqualTo(false);
 197     assertThat(builder.minBlobSize()).isEqualTo(4096);
 198     assertThat(builder.blobFileSize()).isEqualTo(2048);
 199     assertThat(builder.blobCompressionType()).isEqualTo(CompressionType.LZ4_COMPRESSION);
 200     assertThat(builder.enableBlobGarbageCollection()).isEqualTo(false);
 201     assertThat(builder.blobGarbageCollectionAgeCutoff()).isEqualTo(0.91);
 202     assertThat(builder.blobGarbageCollectionForceThreshold()).isEqualTo(0.96);
 203     assertThat(builder.blobCompactionReadaheadSize()).isEqualTo(1024);
 204     assertThat(builder.blobFileStartingLevel()).isEqualTo(0);
 205     assertThat(builder.prepopulateBlobCache())
 206         .isEqualTo(PrepopulateBlobCache.PREPOPULATE_BLOB_DISABLE);
 207
 208     final MutableColumnFamilyOptions options = builder.build();
 209     assertThat(options.getKeys())
 210         .isEqualTo(new String[] {"enable_blob_files", "min_blob_size", "blob_file_size",
 211             "blob_compression_type", "enable_blob_garbage_collection",
 212             "blob_garbage_collection_age_cutoff", "blob_garbage_collection_force_threshold",
 213             "blob_compaction_readahead_size", "blob_file_starting_level",
 214             "prepopulate_blob_cache"});
 215     assertThat(options.getValues())
 216         .isEqualTo(new String[] {"false", "4096", "2048", "LZ4_COMPRESSION", "false", "0.91",
 217             "0.96", "1024", "0", "PREPOPULATE_BLOB_DISABLE"});
 218   }
 219
 220   /**
 221    * Configure the default column family with BLOBs.
 222    * Confirm that BLOBs are generated when appropriately-sized writes are flushed.
 223    *
 224    * @throws RocksDBException if a db access throws an exception
 225    */
 226   @Test
 227   public void testBlobWriteAboveThreshold() throws RocksDBException {
 228     try (final Options options = new Options()
 229                                      .setCreateIfMissing(true)
 230                                      .setMinBlobSize(minBlobSize)
 231                                      .setEnableBlobFiles(true);
 232
 233          final RocksDB db = RocksDB.open(options, dbFolder.getRoot().getAbsolutePath())) {
 234       db.put(small_key("default"), small_value("default"));
 235       db.flush(new FlushOptions().setWaitForFlush(true));
 236
 237       // check there are no blobs in the database
 238       assertThat(countDBFiles(".sst")).isEqualTo(1);
 239       assertThat(countDBFiles(".blob")).isEqualTo(0);
 240
 241       db.put(large_key("default"), large_value("default"));
 242       db.flush(new FlushOptions().setWaitForFlush(true));
 243
 244       // wrote and flushed a value larger than the blobbing threshold
 245       // check there is a single blob in the database
 246       assertThat(countDBFiles(".sst")).isEqualTo(2);
 247       assertThat(countDBFiles(".blob")).isEqualTo(1);
 248
 249       assertThat(db.get(small_key("default"))).isEqualTo(small_value("default"));
 250       assertThat(db.get(large_key("default"))).isEqualTo(large_value("default"));
 251
 252       final MutableColumnFamilyOptions.MutableColumnFamilyOptionsBuilder fetchOptions =
 253           db.getOptions(null);
 254       assertThat(fetchOptions.minBlobSize()).isEqualTo(minBlobSize);
 255       assertThat(fetchOptions.enableBlobFiles()).isEqualTo(true);
 256       assertThat(fetchOptions.writeBufferSize()).isEqualTo(64 << 20);
 257     }
 258   }
 259
 260   /**
 261    * Configure 2 column families respectively with and without BLOBs.
 262    * Confirm that BLOB files are generated (once the DB is flushed) only for the appropriate column
 263    * family.
 264    *
 265    * @throws RocksDBException if a db access throws an exception
 266    */
 267   @Test
 268   public void testBlobWriteAboveThresholdCF() throws RocksDBException {
 269     final ColumnFamilyOptions columnFamilyOptions0 = new ColumnFamilyOptions();
 270     final ColumnFamilyDescriptor columnFamilyDescriptor0 =
 271         new ColumnFamilyDescriptor("default".getBytes(UTF_8), columnFamilyOptions0);
 272     List<ColumnFamilyDescriptor> columnFamilyDescriptors =
 273         Collections.singletonList(columnFamilyDescriptor0);
 274     List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>();
 275
 276     try (final DBOptions dbOptions = new DBOptions().setCreateIfMissing(true);
 277          final RocksDB db = RocksDB.open(dbOptions, dbFolder.getRoot().getAbsolutePath(),
 278              columnFamilyDescriptors, columnFamilyHandles)) {
 279       db.put(columnFamilyHandles.get(0), small_key("default"), small_value("default"));
 280       db.flush(new FlushOptions().setWaitForFlush(true));
 281
 282       assertThat(countDBFiles(".blob")).isEqualTo(0);
 283
 284       try (final ColumnFamilyOptions columnFamilyOptions1 =
 285                new ColumnFamilyOptions().setMinBlobSize(minBlobSize).setEnableBlobFiles(true);
 286
 287            final ColumnFamilyOptions columnFamilyOptions2 =
 288                new ColumnFamilyOptions().setMinBlobSize(minBlobSize).setEnableBlobFiles(false)) {
 289         final ColumnFamilyDescriptor columnFamilyDescriptor1 =
 290             new ColumnFamilyDescriptor("column_family_1".getBytes(UTF_8), columnFamilyOptions1);
 291         final ColumnFamilyDescriptor columnFamilyDescriptor2 =
 292             new ColumnFamilyDescriptor("column_family_2".getBytes(UTF_8), columnFamilyOptions2);
 293
 294         // Create the first column family with blob options
 295         db.createColumnFamily(columnFamilyDescriptor1);
 296
 297         // Create the second column family with not-blob options
 298         db.createColumnFamily(columnFamilyDescriptor2);
 299       }
 300     }
 301
 302     // Now re-open after auto-close - at this point the CF options we use are recognized.
 303     try (final ColumnFamilyOptions columnFamilyOptions1 =
 304              new ColumnFamilyOptions().setMinBlobSize(minBlobSize).setEnableBlobFiles(true);
 305
 306          final ColumnFamilyOptions columnFamilyOptions2 =
 307              new ColumnFamilyOptions().setMinBlobSize(minBlobSize).setEnableBlobFiles(false)) {
 308       assertThat(columnFamilyOptions1.enableBlobFiles()).isEqualTo(true);
 309       assertThat(columnFamilyOptions1.minBlobSize()).isEqualTo(minBlobSize);
 310       assertThat(columnFamilyOptions2.enableBlobFiles()).isEqualTo(false);
 311       assertThat(columnFamilyOptions1.minBlobSize()).isEqualTo(minBlobSize);
 312
 313       final ColumnFamilyDescriptor columnFamilyDescriptor1 =
 314           new ColumnFamilyDescriptor("column_family_1".getBytes(UTF_8), columnFamilyOptions1);
 315       final ColumnFamilyDescriptor columnFamilyDescriptor2 =
 316           new ColumnFamilyDescriptor("column_family_2".getBytes(UTF_8), columnFamilyOptions2);
 317       columnFamilyDescriptors = new ArrayList<>();
 318       columnFamilyDescriptors.add(columnFamilyDescriptor0);
 319       columnFamilyDescriptors.add(columnFamilyDescriptor1);
 320       columnFamilyDescriptors.add(columnFamilyDescriptor2);
 321       columnFamilyHandles = new ArrayList<>();
 322
 323       assertThat(columnFamilyDescriptor1.getOptions().enableBlobFiles()).isEqualTo(true);
 324       assertThat(columnFamilyDescriptor2.getOptions().enableBlobFiles()).isEqualTo(false);
 325
 326       try (final DBOptions dbOptions = new DBOptions();
 327            final RocksDB db = RocksDB.open(dbOptions, dbFolder.getRoot().getAbsolutePath(),
 328                columnFamilyDescriptors, columnFamilyHandles)) {
 329         final MutableColumnFamilyOptions.MutableColumnFamilyOptionsBuilder builder1 =
 330             db.getOptions(columnFamilyHandles.get(1));
 331         assertThat(builder1.enableBlobFiles()).isEqualTo(true);
 332         assertThat(builder1.minBlobSize()).isEqualTo(minBlobSize);
 333
 334         final MutableColumnFamilyOptions.MutableColumnFamilyOptionsBuilder builder2 =
 335             db.getOptions(columnFamilyHandles.get(2));
 336         assertThat(builder2.enableBlobFiles()).isEqualTo(false);
 337         assertThat(builder2.minBlobSize()).isEqualTo(minBlobSize);
 338
 339         db.put(columnFamilyHandles.get(1), large_key("column_family_1_k2"),
 340             large_value("column_family_1_k2"));
 341         db.flush(new FlushOptions().setWaitForFlush(true), columnFamilyHandles.get(1));
 342         assertThat(countDBFiles(".blob")).isEqualTo(1);
 343
 344         db.put(columnFamilyHandles.get(2), large_key("column_family_2_k2"),
 345             large_value("column_family_2_k2"));
 346         db.flush(new FlushOptions().setWaitForFlush(true), columnFamilyHandles.get(2));
 347         assertThat(countDBFiles(".blob")).isEqualTo(1);
 348       }
 349     }
 350   }
 351 }