1 // Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
2 // This source code is licensed under both the GPLv2 (found in the
3 // COPYING file in the root directory) and Apache 2.0 License
4 // (found in the LICENSE.Apache file in the root directory).
6 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
7 // Use of this source code is governed by a BSD-style license that can be
8 // found in the LICENSE file. See the AUTHORS file for names of contributors.
15 #include "db/column_family.h"
16 #include "db/db_iter.h"
17 #include "db/db_test_util.h"
18 #include "db/dbformat.h"
19 #include "db/write_batch_internal.h"
20 #include "port/port.h"
21 #include "port/stack_trace.h"
22 #include "util/string_util.h"
23 #include "utilities/merge_operators.h"
27 // kTypeBlobIndex is a value type used by BlobDB only. The base rocksdb
28 // should accept the value type on write, and report not supported value
29 // for reads, unless caller request for it explicitly. The base rocksdb
30 // doesn't understand format of actual blob index (the value).
31 class DBBlobIndexTest
: public DBTestBase
{
35 kImmutableMemtables
= 1,
39 const std::vector
<Tier
> kAllTiers
= {Tier::kMemtable
,
40 Tier::kImmutableMemtables
,
41 Tier::kL0SstFile
, Tier::kLnSstFile
};
43 DBBlobIndexTest() : DBTestBase("/db_blob_index_test") {}
45 ColumnFamilyHandle
* cfh() { return dbfull()->DefaultColumnFamily(); }
47 ColumnFamilyData
* cfd() {
48 return reinterpret_cast<ColumnFamilyHandleImpl
*>(cfh())->cfd();
51 Status
PutBlobIndex(WriteBatch
* batch
, const Slice
& key
,
52 const Slice
& blob_index
) {
53 return WriteBatchInternal::PutBlobIndex(batch
, cfd()->GetID(), key
,
57 Status
Write(WriteBatch
* batch
) {
58 return dbfull()->Write(WriteOptions(), batch
);
61 std::string
GetImpl(const Slice
& key
, bool* is_blob_index
= nullptr,
62 const Snapshot
* snapshot
= nullptr) {
63 ReadOptions read_options
;
64 read_options
.snapshot
= snapshot
;
66 auto s
= dbfull()->GetImpl(read_options
, cfh(), key
, &value
,
67 nullptr /*value_found*/, nullptr /*callback*/,
72 if (s
.IsNotSupported()) {
73 return "NOT_SUPPORTED";
78 return value
.ToString();
81 std::string
GetBlobIndex(const Slice
& key
,
82 const Snapshot
* snapshot
= nullptr) {
83 bool is_blob_index
= false;
84 std::string value
= GetImpl(key
, &is_blob_index
, snapshot
);
91 ArenaWrappedDBIter
* GetBlobIterator() {
92 return dbfull()->NewIteratorImpl(
93 ReadOptions(), cfd(), dbfull()->GetLatestSequenceNumber(),
94 nullptr /*read_callback*/, true /*allow_blob*/);
97 Options
GetTestOptions() {
99 options
.create_if_missing
= true;
100 options
.num_levels
= 2;
101 options
.disable_auto_compactions
= true;
102 // Disable auto flushes.
103 options
.max_write_buffer_number
= 10;
104 options
.min_write_buffer_number_to_merge
= 10;
105 options
.merge_operator
= MergeOperators::CreateStringAppendOperator();
109 void MoveDataTo(Tier tier
) {
111 case Tier::kMemtable
:
113 case Tier::kImmutableMemtables
:
114 ASSERT_OK(dbfull()->TEST_SwitchMemtable());
116 case Tier::kL0SstFile
:
119 case Tier::kLnSstFile
:
121 ASSERT_OK(Put("a", "dummy"));
122 ASSERT_OK(Put("z", "dummy"));
125 dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr));
127 ASSERT_EQ("0,1", FilesPerLevel());
128 #endif // !ROCKSDB_LITE
134 // Should be able to write kTypeBlobIndex to memtables and SST files.
135 TEST_F(DBBlobIndexTest
, Write
) {
136 for (auto tier
: kAllTiers
) {
137 DestroyAndReopen(GetTestOptions());
138 for (int i
= 1; i
<= 5; i
++) {
139 std::string index
= ToString(i
);
141 ASSERT_OK(PutBlobIndex(&batch
, "key" + index
, "blob" + index
));
142 ASSERT_OK(Write(&batch
));
145 for (int i
= 1; i
<= 5; i
++) {
146 std::string index
= ToString(i
);
147 ASSERT_EQ("blob" + index
, GetBlobIndex("key" + index
));
152 // Get should be able to return blob index if is_blob_index is provided,
153 // otherwise return Status::NotSupported status.
154 TEST_F(DBBlobIndexTest
, Get
) {
155 for (auto tier
: kAllTiers
) {
156 DestroyAndReopen(GetTestOptions());
158 ASSERT_OK(batch
.Put("key", "value"));
159 ASSERT_OK(PutBlobIndex(&batch
, "blob_key", "blob_index"));
160 ASSERT_OK(Write(&batch
));
162 // Verify normal value
163 bool is_blob_index
= false;
165 ASSERT_EQ("value", Get("key"));
166 ASSERT_EQ("value", GetImpl("key"));
167 ASSERT_EQ("value", GetImpl("key", &is_blob_index
));
168 ASSERT_FALSE(is_blob_index
);
170 ASSERT_TRUE(Get("blob_key", &value
).IsNotSupported());
171 ASSERT_EQ("NOT_SUPPORTED", GetImpl("blob_key"));
172 ASSERT_EQ("blob_index", GetImpl("blob_key", &is_blob_index
));
173 ASSERT_TRUE(is_blob_index
);
177 // Get should NOT return Status::NotSupported if blob index is updated with
179 TEST_F(DBBlobIndexTest
, Updated
) {
180 for (auto tier
: kAllTiers
) {
181 DestroyAndReopen(GetTestOptions());
183 for (int i
= 0; i
< 10; i
++) {
184 ASSERT_OK(PutBlobIndex(&batch
, "key" + ToString(i
), "blob_index"));
186 ASSERT_OK(Write(&batch
));
187 // Avoid blob values from being purged.
188 const Snapshot
* snapshot
= dbfull()->GetSnapshot();
189 ASSERT_OK(Put("key1", "new_value"));
190 ASSERT_OK(Merge("key2", "a"));
191 ASSERT_OK(Merge("key2", "b"));
192 ASSERT_OK(Merge("key2", "c"));
193 ASSERT_OK(Delete("key3"));
194 ASSERT_OK(SingleDelete("key4"));
195 ASSERT_OK(Delete("key5"));
196 ASSERT_OK(Merge("key5", "a"));
197 ASSERT_OK(Merge("key5", "b"));
198 ASSERT_OK(Merge("key5", "c"));
199 ASSERT_OK(dbfull()->DeleteRange(WriteOptions(), cfh(), "key6", "key9"));
201 for (int i
= 0; i
< 10; i
++) {
202 ASSERT_EQ("blob_index", GetBlobIndex("key" + ToString(i
), snapshot
));
204 ASSERT_EQ("new_value", Get("key1"));
205 ASSERT_EQ("NOT_SUPPORTED", GetImpl("key2"));
206 ASSERT_EQ("NOT_FOUND", Get("key3"));
207 ASSERT_EQ("NOT_FOUND", Get("key4"));
208 ASSERT_EQ("a,b,c", GetImpl("key5"));
209 for (int i
= 6; i
< 9; i
++) {
210 ASSERT_EQ("NOT_FOUND", Get("key" + ToString(i
)));
212 ASSERT_EQ("blob_index", GetBlobIndex("key9"));
213 dbfull()->ReleaseSnapshot(snapshot
);
217 // Iterator should get blob value if allow_blob flag is set,
218 // otherwise return Status::NotSupported status.
219 TEST_F(DBBlobIndexTest
, Iterate
) {
220 const std::vector
<std::vector
<ValueType
>> data
= {
222 /*01*/ {kTypeBlobIndex
},
224 /*03*/ {kTypeBlobIndex
, kTypeValue
},
226 /*05*/ {kTypeValue
, kTypeBlobIndex
},
228 /*07*/ {kTypeDeletion
, kTypeBlobIndex
},
230 /*09*/ {kTypeSingleDeletion
, kTypeBlobIndex
},
232 /*11*/ {kTypeMerge
, kTypeMerge
, kTypeMerge
, kTypeBlobIndex
},
235 {kTypeMerge
, kTypeMerge
, kTypeMerge
, kTypeDeletion
, kTypeBlobIndex
},
237 /*15*/ {kTypeBlobIndex
},
241 auto get_key
= [](int index
) {
243 snprintf(buf
, sizeof(buf
), "%02d", index
);
244 return "key" + std::string(buf
);
247 auto get_value
= [&](int index
, int version
) {
248 return get_key(index
) + "_value" + ToString(version
);
251 auto check_iterator
= [&](Iterator
* iterator
, Status::Code expected_status
,
252 const Slice
& expected_value
) {
253 ASSERT_EQ(expected_status
, iterator
->status().code());
254 if (expected_status
== Status::kOk
) {
255 ASSERT_TRUE(iterator
->Valid());
256 ASSERT_EQ(expected_value
, iterator
->value());
258 ASSERT_FALSE(iterator
->Valid());
262 auto create_normal_iterator
= [&]() -> Iterator
* {
263 return dbfull()->NewIterator(ReadOptions());
266 auto create_blob_iterator
= [&]() -> Iterator
* { return GetBlobIterator(); };
268 auto check_is_blob
= [&](bool is_blob
) {
269 return [is_blob
](Iterator
* iterator
) {
271 reinterpret_cast<ArenaWrappedDBIter
*>(iterator
)->IsBlob());
275 auto verify
= [&](int index
, Status::Code expected_status
,
276 const Slice
& forward_value
, const Slice
& backward_value
,
277 std::function
<Iterator
*()> create_iterator
,
278 std::function
<void(Iterator
*)> extra_check
= nullptr) {
280 auto* iterator
= create_iterator();
281 ASSERT_OK(iterator
->Refresh());
282 iterator
->Seek(get_key(index
));
283 check_iterator(iterator
, expected_status
, forward_value
);
285 extra_check(iterator
);
290 iterator
= create_iterator();
291 ASSERT_OK(iterator
->Refresh());
292 iterator
->Seek(get_key(index
- 1));
293 ASSERT_TRUE(iterator
->Valid());
295 check_iterator(iterator
, expected_status
, forward_value
);
297 extra_check(iterator
);
302 iterator
= create_iterator();
303 ASSERT_OK(iterator
->Refresh());
304 iterator
->SeekForPrev(get_key(index
));
305 check_iterator(iterator
, expected_status
, backward_value
);
307 extra_check(iterator
);
312 iterator
= create_iterator();
313 iterator
->Seek(get_key(index
+ 1));
314 ASSERT_TRUE(iterator
->Valid());
316 check_iterator(iterator
, expected_status
, backward_value
);
318 extra_check(iterator
);
323 for (auto tier
: {Tier::kMemtable
} /*kAllTiers*/) {
324 // Avoid values from being purged.
325 std::vector
<const Snapshot
*> snapshots
;
326 DestroyAndReopen(GetTestOptions());
329 for (int i
= 0; i
< static_cast<int>(data
.size()); i
++) {
330 for (int j
= static_cast<int>(data
[i
].size()) - 1; j
>= 0; j
--) {
331 std::string key
= get_key(i
);
332 std::string value
= get_value(i
, j
);
334 switch (data
[i
][j
]) {
336 ASSERT_OK(Put(key
, value
));
339 ASSERT_OK(Delete(key
));
341 case kTypeSingleDeletion
:
342 ASSERT_OK(SingleDelete(key
));
345 ASSERT_OK(Merge(key
, value
));
348 ASSERT_OK(PutBlobIndex(&batch
, key
, value
));
349 ASSERT_OK(Write(&batch
));
355 snapshots
.push_back(dbfull()->GetSnapshot());
358 dbfull()->DeleteRange(WriteOptions(), cfh(), get_key(15), get_key(16)));
359 snapshots
.push_back(dbfull()->GetSnapshot());
363 verify(1, Status::kNotSupported
, "", "", create_normal_iterator
);
364 verify(3, Status::kNotSupported
, "", "", create_normal_iterator
);
365 verify(5, Status::kOk
, get_value(5, 0), get_value(5, 0),
366 create_normal_iterator
);
367 verify(7, Status::kOk
, get_value(8, 0), get_value(6, 0),
368 create_normal_iterator
);
369 verify(9, Status::kOk
, get_value(10, 0), get_value(8, 0),
370 create_normal_iterator
);
371 verify(11, Status::kNotSupported
, "", "", create_normal_iterator
);
372 verify(13, Status::kOk
,
373 get_value(13, 2) + "," + get_value(13, 1) + "," + get_value(13, 0),
374 get_value(13, 2) + "," + get_value(13, 1) + "," + get_value(13, 0),
375 create_normal_iterator
);
376 verify(15, Status::kOk
, get_value(16, 0), get_value(14, 0),
377 create_normal_iterator
);
379 // Iterator with blob support
380 verify(1, Status::kOk
, get_value(1, 0), get_value(1, 0),
381 create_blob_iterator
, check_is_blob(true));
382 verify(3, Status::kOk
, get_value(3, 0), get_value(3, 0),
383 create_blob_iterator
, check_is_blob(true));
384 verify(5, Status::kOk
, get_value(5, 0), get_value(5, 0),
385 create_blob_iterator
, check_is_blob(false));
386 verify(7, Status::kOk
, get_value(8, 0), get_value(6, 0),
387 create_blob_iterator
, check_is_blob(false));
388 verify(9, Status::kOk
, get_value(10, 0), get_value(8, 0),
389 create_blob_iterator
, check_is_blob(false));
390 verify(11, Status::kNotSupported
, "", "", create_blob_iterator
);
391 verify(13, Status::kOk
,
392 get_value(13, 2) + "," + get_value(13, 1) + "," + get_value(13, 0),
393 get_value(13, 2) + "," + get_value(13, 1) + "," + get_value(13, 0),
394 create_blob_iterator
, check_is_blob(false));
395 verify(15, Status::kOk
, get_value(16, 0), get_value(14, 0),
396 create_blob_iterator
, check_is_blob(false));
398 for (auto* snapshot
: snapshots
) {
399 dbfull()->ReleaseSnapshot(snapshot
);
404 } // namespace rocksdb
406 int main(int argc
, char** argv
) {
407 rocksdb::port::InstallStackTraceHandler();
408 ::testing::InitGoogleTest(&argc
, argv
);
409 return RUN_ALL_TESTS();