]>
Commit | Line | Data |
---|---|---|
1e59de90 TL |
1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. |
2 | // This source code is licensed under both the GPLv2 (found in the | |
3 | // COPYING file in the root directory) and Apache 2.0 License | |
4 | // (found in the LICENSE.Apache file in the root directory). | |
5 | package org.rocksdb; | |
6 | ||
7 | import static java.nio.charset.StandardCharsets.UTF_8; | |
8 | import static org.assertj.core.api.Assertions.assertThat; | |
9 | ||
10 | import java.io.File; | |
11 | import java.io.FilenameFilter; | |
12 | import java.util.*; | |
13 | import org.junit.ClassRule; | |
14 | import org.junit.Rule; | |
15 | import org.junit.Test; | |
16 | import org.junit.rules.TemporaryFolder; | |
17 | ||
18 | public class BlobOptionsTest { | |
19 | @ClassRule | |
20 | public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = | |
21 | new RocksNativeLibraryResource(); | |
22 | ||
23 | @Rule public TemporaryFolder dbFolder = new TemporaryFolder(); | |
24 | ||
25 | final int minBlobSize = 65536; | |
26 | final int largeBlobSize = 65536 * 2; | |
27 | ||
28 | /** | |
29 | * Count the files in the temporary folder which end with a particular suffix | |
30 | * Used to query the state of a test database to check if it is as the test expects | |
31 | * | |
32 | * @param endsWith the suffix to match | |
33 | * @return the number of files with a matching suffix | |
34 | */ | |
35 | @SuppressWarnings("CallToStringConcatCanBeReplacedByOperator") | |
36 | private int countDBFiles(final String endsWith) { | |
37 | return Objects | |
38 | .requireNonNull(dbFolder.getRoot().list(new FilenameFilter() { | |
39 | @Override | |
40 | public boolean accept(File dir, String name) { | |
41 | return name.endsWith(endsWith); | |
42 | } | |
43 | })) | |
44 | .length; | |
45 | } | |
46 | ||
47 | @SuppressWarnings("SameParameterValue") | |
48 | private byte[] small_key(String suffix) { | |
49 | return ("small_key_" + suffix).getBytes(UTF_8); | |
50 | } | |
51 | ||
52 | @SuppressWarnings("SameParameterValue") | |
53 | private byte[] small_value(String suffix) { | |
54 | return ("small_value_" + suffix).getBytes(UTF_8); | |
55 | } | |
56 | ||
57 | private byte[] large_key(String suffix) { | |
58 | return ("large_key_" + suffix).getBytes(UTF_8); | |
59 | } | |
60 | ||
61 | private byte[] large_value(String repeat) { | |
62 | final byte[] large_value = ("" + repeat + "_" + largeBlobSize + "b").getBytes(UTF_8); | |
63 | final byte[] large_buffer = new byte[largeBlobSize]; | |
64 | for (int pos = 0; pos < largeBlobSize; pos += large_value.length) { | |
65 | int numBytes = Math.min(large_value.length, large_buffer.length - pos); | |
66 | System.arraycopy(large_value, 0, large_buffer, pos, numBytes); | |
67 | } | |
68 | return large_buffer; | |
69 | } | |
70 | ||
71 | @Test | |
72 | public void blobOptions() { | |
73 | try (final Options options = new Options()) { | |
74 | assertThat(options.enableBlobFiles()).isEqualTo(false); | |
75 | assertThat(options.minBlobSize()).isEqualTo(0); | |
76 | assertThat(options.blobCompressionType()).isEqualTo(CompressionType.NO_COMPRESSION); | |
77 | assertThat(options.enableBlobGarbageCollection()).isEqualTo(false); | |
78 | assertThat(options.blobFileSize()).isEqualTo(268435456L); | |
79 | assertThat(options.blobGarbageCollectionAgeCutoff()).isEqualTo(0.25); | |
80 | assertThat(options.blobGarbageCollectionForceThreshold()).isEqualTo(1.0); | |
81 | assertThat(options.blobCompactionReadaheadSize()).isEqualTo(0); | |
82 | assertThat(options.prepopulateBlobCache()) | |
83 | .isEqualTo(PrepopulateBlobCache.PREPOPULATE_BLOB_DISABLE); | |
84 | ||
85 | assertThat(options.setEnableBlobFiles(true)).isEqualTo(options); | |
86 | assertThat(options.setMinBlobSize(132768L)).isEqualTo(options); | |
87 | assertThat(options.setBlobCompressionType(CompressionType.BZLIB2_COMPRESSION)) | |
88 | .isEqualTo(options); | |
89 | assertThat(options.setEnableBlobGarbageCollection(true)).isEqualTo(options); | |
90 | assertThat(options.setBlobFileSize(132768L)).isEqualTo(options); | |
91 | assertThat(options.setBlobGarbageCollectionAgeCutoff(0.89)).isEqualTo(options); | |
92 | assertThat(options.setBlobGarbageCollectionForceThreshold(0.80)).isEqualTo(options); | |
93 | assertThat(options.setBlobCompactionReadaheadSize(262144L)).isEqualTo(options); | |
94 | assertThat(options.setBlobFileStartingLevel(0)).isEqualTo(options); | |
95 | assertThat(options.setPrepopulateBlobCache(PrepopulateBlobCache.PREPOPULATE_BLOB_FLUSH_ONLY)) | |
96 | .isEqualTo(options); | |
97 | ||
98 | assertThat(options.enableBlobFiles()).isEqualTo(true); | |
99 | assertThat(options.minBlobSize()).isEqualTo(132768L); | |
100 | assertThat(options.blobCompressionType()).isEqualTo(CompressionType.BZLIB2_COMPRESSION); | |
101 | assertThat(options.enableBlobGarbageCollection()).isEqualTo(true); | |
102 | assertThat(options.blobFileSize()).isEqualTo(132768L); | |
103 | assertThat(options.blobGarbageCollectionAgeCutoff()).isEqualTo(0.89); | |
104 | assertThat(options.blobGarbageCollectionForceThreshold()).isEqualTo(0.80); | |
105 | assertThat(options.blobCompactionReadaheadSize()).isEqualTo(262144L); | |
106 | assertThat(options.blobFileStartingLevel()).isEqualTo(0); | |
107 | assertThat(options.prepopulateBlobCache()) | |
108 | .isEqualTo(PrepopulateBlobCache.PREPOPULATE_BLOB_FLUSH_ONLY); | |
109 | } | |
110 | } | |
111 | ||
112 | @Test | |
113 | public void blobColumnFamilyOptions() { | |
114 | try (final ColumnFamilyOptions columnFamilyOptions = new ColumnFamilyOptions()) { | |
115 | assertThat(columnFamilyOptions.enableBlobFiles()).isEqualTo(false); | |
116 | assertThat(columnFamilyOptions.minBlobSize()).isEqualTo(0); | |
117 | assertThat(columnFamilyOptions.blobCompressionType()) | |
118 | .isEqualTo(CompressionType.NO_COMPRESSION); | |
119 | assertThat(columnFamilyOptions.enableBlobGarbageCollection()).isEqualTo(false); | |
120 | assertThat(columnFamilyOptions.blobFileSize()).isEqualTo(268435456L); | |
121 | assertThat(columnFamilyOptions.blobGarbageCollectionAgeCutoff()).isEqualTo(0.25); | |
122 | assertThat(columnFamilyOptions.blobGarbageCollectionForceThreshold()).isEqualTo(1.0); | |
123 | assertThat(columnFamilyOptions.blobCompactionReadaheadSize()).isEqualTo(0); | |
124 | ||
125 | assertThat(columnFamilyOptions.setEnableBlobFiles(true)).isEqualTo(columnFamilyOptions); | |
126 | assertThat(columnFamilyOptions.setMinBlobSize(132768L)).isEqualTo(columnFamilyOptions); | |
127 | assertThat(columnFamilyOptions.setBlobCompressionType(CompressionType.BZLIB2_COMPRESSION)) | |
128 | .isEqualTo(columnFamilyOptions); | |
129 | assertThat(columnFamilyOptions.setEnableBlobGarbageCollection(true)) | |
130 | .isEqualTo(columnFamilyOptions); | |
131 | assertThat(columnFamilyOptions.setBlobFileSize(132768L)).isEqualTo(columnFamilyOptions); | |
132 | assertThat(columnFamilyOptions.setBlobGarbageCollectionAgeCutoff(0.89)) | |
133 | .isEqualTo(columnFamilyOptions); | |
134 | assertThat(columnFamilyOptions.setBlobGarbageCollectionForceThreshold(0.80)) | |
135 | .isEqualTo(columnFamilyOptions); | |
136 | assertThat(columnFamilyOptions.setBlobCompactionReadaheadSize(262144L)) | |
137 | .isEqualTo(columnFamilyOptions); | |
138 | assertThat(columnFamilyOptions.setBlobFileStartingLevel(0)).isEqualTo(columnFamilyOptions); | |
139 | assertThat(columnFamilyOptions.setPrepopulateBlobCache( | |
140 | PrepopulateBlobCache.PREPOPULATE_BLOB_DISABLE)) | |
141 | .isEqualTo(columnFamilyOptions); | |
142 | ||
143 | assertThat(columnFamilyOptions.enableBlobFiles()).isEqualTo(true); | |
144 | assertThat(columnFamilyOptions.minBlobSize()).isEqualTo(132768L); | |
145 | assertThat(columnFamilyOptions.blobCompressionType()) | |
146 | .isEqualTo(CompressionType.BZLIB2_COMPRESSION); | |
147 | assertThat(columnFamilyOptions.enableBlobGarbageCollection()).isEqualTo(true); | |
148 | assertThat(columnFamilyOptions.blobFileSize()).isEqualTo(132768L); | |
149 | assertThat(columnFamilyOptions.blobGarbageCollectionAgeCutoff()).isEqualTo(0.89); | |
150 | assertThat(columnFamilyOptions.blobGarbageCollectionForceThreshold()).isEqualTo(0.80); | |
151 | assertThat(columnFamilyOptions.blobCompactionReadaheadSize()).isEqualTo(262144L); | |
152 | assertThat(columnFamilyOptions.blobFileStartingLevel()).isEqualTo(0); | |
153 | assertThat(columnFamilyOptions.prepopulateBlobCache()) | |
154 | .isEqualTo(PrepopulateBlobCache.PREPOPULATE_BLOB_DISABLE); | |
155 | } | |
156 | } | |
157 | ||
158 | @Test | |
159 | public void blobMutableColumnFamilyOptionsBuilder() { | |
160 | final MutableColumnFamilyOptions.MutableColumnFamilyOptionsBuilder builder = | |
161 | MutableColumnFamilyOptions.builder(); | |
162 | builder.setEnableBlobFiles(true) | |
163 | .setMinBlobSize(1024) | |
164 | .setBlobFileSize(132768) | |
165 | .setBlobCompressionType(CompressionType.BZLIB2_COMPRESSION) | |
166 | .setEnableBlobGarbageCollection(true) | |
167 | .setBlobGarbageCollectionAgeCutoff(0.89) | |
168 | .setBlobGarbageCollectionForceThreshold(0.80) | |
169 | .setBlobCompactionReadaheadSize(262144) | |
170 | .setBlobFileStartingLevel(1) | |
171 | .setPrepopulateBlobCache(PrepopulateBlobCache.PREPOPULATE_BLOB_FLUSH_ONLY); | |
172 | ||
173 | assertThat(builder.enableBlobFiles()).isEqualTo(true); | |
174 | assertThat(builder.minBlobSize()).isEqualTo(1024); | |
175 | assertThat(builder.blobFileSize()).isEqualTo(132768); | |
176 | assertThat(builder.blobCompressionType()).isEqualTo(CompressionType.BZLIB2_COMPRESSION); | |
177 | assertThat(builder.enableBlobGarbageCollection()).isEqualTo(true); | |
178 | assertThat(builder.blobGarbageCollectionAgeCutoff()).isEqualTo(0.89); | |
179 | assertThat(builder.blobGarbageCollectionForceThreshold()).isEqualTo(0.80); | |
180 | assertThat(builder.blobCompactionReadaheadSize()).isEqualTo(262144); | |
181 | assertThat(builder.blobFileStartingLevel()).isEqualTo(1); | |
182 | assertThat(builder.prepopulateBlobCache()) | |
183 | .isEqualTo(PrepopulateBlobCache.PREPOPULATE_BLOB_FLUSH_ONLY); | |
184 | ||
185 | builder.setEnableBlobFiles(false) | |
186 | .setMinBlobSize(4096) | |
187 | .setBlobFileSize(2048) | |
188 | .setBlobCompressionType(CompressionType.LZ4_COMPRESSION) | |
189 | .setEnableBlobGarbageCollection(false) | |
190 | .setBlobGarbageCollectionAgeCutoff(0.91) | |
191 | .setBlobGarbageCollectionForceThreshold(0.96) | |
192 | .setBlobCompactionReadaheadSize(1024) | |
193 | .setBlobFileStartingLevel(0) | |
194 | .setPrepopulateBlobCache(PrepopulateBlobCache.PREPOPULATE_BLOB_DISABLE); | |
195 | ||
196 | assertThat(builder.enableBlobFiles()).isEqualTo(false); | |
197 | assertThat(builder.minBlobSize()).isEqualTo(4096); | |
198 | assertThat(builder.blobFileSize()).isEqualTo(2048); | |
199 | assertThat(builder.blobCompressionType()).isEqualTo(CompressionType.LZ4_COMPRESSION); | |
200 | assertThat(builder.enableBlobGarbageCollection()).isEqualTo(false); | |
201 | assertThat(builder.blobGarbageCollectionAgeCutoff()).isEqualTo(0.91); | |
202 | assertThat(builder.blobGarbageCollectionForceThreshold()).isEqualTo(0.96); | |
203 | assertThat(builder.blobCompactionReadaheadSize()).isEqualTo(1024); | |
204 | assertThat(builder.blobFileStartingLevel()).isEqualTo(0); | |
205 | assertThat(builder.prepopulateBlobCache()) | |
206 | .isEqualTo(PrepopulateBlobCache.PREPOPULATE_BLOB_DISABLE); | |
207 | ||
208 | final MutableColumnFamilyOptions options = builder.build(); | |
209 | assertThat(options.getKeys()) | |
210 | .isEqualTo(new String[] {"enable_blob_files", "min_blob_size", "blob_file_size", | |
211 | "blob_compression_type", "enable_blob_garbage_collection", | |
212 | "blob_garbage_collection_age_cutoff", "blob_garbage_collection_force_threshold", | |
213 | "blob_compaction_readahead_size", "blob_file_starting_level", | |
214 | "prepopulate_blob_cache"}); | |
215 | assertThat(options.getValues()) | |
216 | .isEqualTo(new String[] {"false", "4096", "2048", "LZ4_COMPRESSION", "false", "0.91", | |
217 | "0.96", "1024", "0", "PREPOPULATE_BLOB_DISABLE"}); | |
218 | } | |
219 | ||
220 | /** | |
221 | * Configure the default column family with BLOBs. | |
222 | * Confirm that BLOBs are generated when appropriately-sized writes are flushed. | |
223 | * | |
224 | * @throws RocksDBException if a db access throws an exception | |
225 | */ | |
226 | @Test | |
227 | public void testBlobWriteAboveThreshold() throws RocksDBException { | |
228 | try (final Options options = new Options() | |
229 | .setCreateIfMissing(true) | |
230 | .setMinBlobSize(minBlobSize) | |
231 | .setEnableBlobFiles(true); | |
232 | ||
233 | final RocksDB db = RocksDB.open(options, dbFolder.getRoot().getAbsolutePath())) { | |
234 | db.put(small_key("default"), small_value("default")); | |
235 | db.flush(new FlushOptions().setWaitForFlush(true)); | |
236 | ||
237 | // check there are no blobs in the database | |
238 | assertThat(countDBFiles(".sst")).isEqualTo(1); | |
239 | assertThat(countDBFiles(".blob")).isEqualTo(0); | |
240 | ||
241 | db.put(large_key("default"), large_value("default")); | |
242 | db.flush(new FlushOptions().setWaitForFlush(true)); | |
243 | ||
244 | // wrote and flushed a value larger than the blobbing threshold | |
245 | // check there is a single blob in the database | |
246 | assertThat(countDBFiles(".sst")).isEqualTo(2); | |
247 | assertThat(countDBFiles(".blob")).isEqualTo(1); | |
248 | ||
249 | assertThat(db.get(small_key("default"))).isEqualTo(small_value("default")); | |
250 | assertThat(db.get(large_key("default"))).isEqualTo(large_value("default")); | |
251 | ||
252 | final MutableColumnFamilyOptions.MutableColumnFamilyOptionsBuilder fetchOptions = | |
253 | db.getOptions(null); | |
254 | assertThat(fetchOptions.minBlobSize()).isEqualTo(minBlobSize); | |
255 | assertThat(fetchOptions.enableBlobFiles()).isEqualTo(true); | |
256 | assertThat(fetchOptions.writeBufferSize()).isEqualTo(64 << 20); | |
257 | } | |
258 | } | |
259 | ||
260 | /** | |
261 | * Configure 2 column families respectively with and without BLOBs. | |
262 | * Confirm that BLOB files are generated (once the DB is flushed) only for the appropriate column | |
263 | * family. | |
264 | * | |
265 | * @throws RocksDBException if a db access throws an exception | |
266 | */ | |
267 | @Test | |
268 | public void testBlobWriteAboveThresholdCF() throws RocksDBException { | |
269 | final ColumnFamilyOptions columnFamilyOptions0 = new ColumnFamilyOptions(); | |
270 | final ColumnFamilyDescriptor columnFamilyDescriptor0 = | |
271 | new ColumnFamilyDescriptor("default".getBytes(UTF_8), columnFamilyOptions0); | |
272 | List<ColumnFamilyDescriptor> columnFamilyDescriptors = | |
273 | Collections.singletonList(columnFamilyDescriptor0); | |
274 | List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>(); | |
275 | ||
276 | try (final DBOptions dbOptions = new DBOptions().setCreateIfMissing(true); | |
277 | final RocksDB db = RocksDB.open(dbOptions, dbFolder.getRoot().getAbsolutePath(), | |
278 | columnFamilyDescriptors, columnFamilyHandles)) { | |
279 | db.put(columnFamilyHandles.get(0), small_key("default"), small_value("default")); | |
280 | db.flush(new FlushOptions().setWaitForFlush(true)); | |
281 | ||
282 | assertThat(countDBFiles(".blob")).isEqualTo(0); | |
283 | ||
284 | try (final ColumnFamilyOptions columnFamilyOptions1 = | |
285 | new ColumnFamilyOptions().setMinBlobSize(minBlobSize).setEnableBlobFiles(true); | |
286 | ||
287 | final ColumnFamilyOptions columnFamilyOptions2 = | |
288 | new ColumnFamilyOptions().setMinBlobSize(minBlobSize).setEnableBlobFiles(false)) { | |
289 | final ColumnFamilyDescriptor columnFamilyDescriptor1 = | |
290 | new ColumnFamilyDescriptor("column_family_1".getBytes(UTF_8), columnFamilyOptions1); | |
291 | final ColumnFamilyDescriptor columnFamilyDescriptor2 = | |
292 | new ColumnFamilyDescriptor("column_family_2".getBytes(UTF_8), columnFamilyOptions2); | |
293 | ||
294 | // Create the first column family with blob options | |
295 | db.createColumnFamily(columnFamilyDescriptor1); | |
296 | ||
297 | // Create the second column family with not-blob options | |
298 | db.createColumnFamily(columnFamilyDescriptor2); | |
299 | } | |
300 | } | |
301 | ||
302 | // Now re-open after auto-close - at this point the CF options we use are recognized. | |
303 | try (final ColumnFamilyOptions columnFamilyOptions1 = | |
304 | new ColumnFamilyOptions().setMinBlobSize(minBlobSize).setEnableBlobFiles(true); | |
305 | ||
306 | final ColumnFamilyOptions columnFamilyOptions2 = | |
307 | new ColumnFamilyOptions().setMinBlobSize(minBlobSize).setEnableBlobFiles(false)) { | |
308 | assertThat(columnFamilyOptions1.enableBlobFiles()).isEqualTo(true); | |
309 | assertThat(columnFamilyOptions1.minBlobSize()).isEqualTo(minBlobSize); | |
310 | assertThat(columnFamilyOptions2.enableBlobFiles()).isEqualTo(false); | |
311 | assertThat(columnFamilyOptions1.minBlobSize()).isEqualTo(minBlobSize); | |
312 | ||
313 | final ColumnFamilyDescriptor columnFamilyDescriptor1 = | |
314 | new ColumnFamilyDescriptor("column_family_1".getBytes(UTF_8), columnFamilyOptions1); | |
315 | final ColumnFamilyDescriptor columnFamilyDescriptor2 = | |
316 | new ColumnFamilyDescriptor("column_family_2".getBytes(UTF_8), columnFamilyOptions2); | |
317 | columnFamilyDescriptors = new ArrayList<>(); | |
318 | columnFamilyDescriptors.add(columnFamilyDescriptor0); | |
319 | columnFamilyDescriptors.add(columnFamilyDescriptor1); | |
320 | columnFamilyDescriptors.add(columnFamilyDescriptor2); | |
321 | columnFamilyHandles = new ArrayList<>(); | |
322 | ||
323 | assertThat(columnFamilyDescriptor1.getOptions().enableBlobFiles()).isEqualTo(true); | |
324 | assertThat(columnFamilyDescriptor2.getOptions().enableBlobFiles()).isEqualTo(false); | |
325 | ||
326 | try (final DBOptions dbOptions = new DBOptions(); | |
327 | final RocksDB db = RocksDB.open(dbOptions, dbFolder.getRoot().getAbsolutePath(), | |
328 | columnFamilyDescriptors, columnFamilyHandles)) { | |
329 | final MutableColumnFamilyOptions.MutableColumnFamilyOptionsBuilder builder1 = | |
330 | db.getOptions(columnFamilyHandles.get(1)); | |
331 | assertThat(builder1.enableBlobFiles()).isEqualTo(true); | |
332 | assertThat(builder1.minBlobSize()).isEqualTo(minBlobSize); | |
333 | ||
334 | final MutableColumnFamilyOptions.MutableColumnFamilyOptionsBuilder builder2 = | |
335 | db.getOptions(columnFamilyHandles.get(2)); | |
336 | assertThat(builder2.enableBlobFiles()).isEqualTo(false); | |
337 | assertThat(builder2.minBlobSize()).isEqualTo(minBlobSize); | |
338 | ||
339 | db.put(columnFamilyHandles.get(1), large_key("column_family_1_k2"), | |
340 | large_value("column_family_1_k2")); | |
341 | db.flush(new FlushOptions().setWaitForFlush(true), columnFamilyHandles.get(1)); | |
342 | assertThat(countDBFiles(".blob")).isEqualTo(1); | |
343 | ||
344 | db.put(columnFamilyHandles.get(2), large_key("column_family_2_k2"), | |
345 | large_value("column_family_2_k2")); | |
346 | db.flush(new FlushOptions().setWaitForFlush(true), columnFamilyHandles.get(2)); | |
347 | assertThat(countDBFiles(".blob")).isEqualTo(1); | |
348 | } | |
349 | } | |
350 | } | |
351 | } |