1 // Use of this source code is governed by a BSD-style license that can be
2 // found in the LICENSE file. See the AUTHORS file for names of contributors.
3 // Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
4 // This source code is licensed under the BSD-style license found in the
5 // LICENSE file in the root directory of this source tree. An additional grant
6 // of patent rights can be found in the PATENTS file in the same directory.
8 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
9 // Use of this source code is governed by a BSD-style license that can be
10 // found in the LICENSE file. See the AUTHORS file for names of contributors.
17 #include "db/db_impl.h"
18 #include "db/version_set.h"
19 #include "db/write_batch_internal.h"
20 #include "rocksdb/cache.h"
21 #include "rocksdb/compaction_filter.h"
22 #include "rocksdb/db.h"
23 #include "rocksdb/env.h"
24 #include "rocksdb/filter_policy.h"
25 #include "rocksdb/slice_transform.h"
26 #include "rocksdb/table.h"
27 #include "table/bloom_block.h"
28 #include "table/meta_blocks.h"
29 #include "table/plain_table_factory.h"
30 #include "table/plain_table_key_coding.h"
31 #include "table/plain_table_reader.h"
32 #include "table/table_builder.h"
33 #include "util/filename.h"
34 #include "util/hash.h"
35 #include "util/logging.h"
36 #include "util/mutexlock.h"
37 #include "util/string_util.h"
38 #include "util/testharness.h"
39 #include "util/testutil.h"
40 #include "utilities/merge_operators.h"
42 using std::unique_ptr
;
45 class PlainTableKeyDecoderTest
: public testing::Test
{};
47 TEST_F(PlainTableKeyDecoderTest
, ReadNonMmap
) {
50 const uint32_t kLength
= 2222;
51 Slice contents
= test::RandomString(&rnd
, kLength
, &tmp
);
52 test::StringSource
* string_source
=
53 new test::StringSource(contents
, 0, false);
55 unique_ptr
<RandomAccessFileReader
> file_reader(
56 test::GetRandomAccessFileReader(string_source
));
57 unique_ptr
<PlainTableReaderFileInfo
> file_info(new PlainTableReaderFileInfo(
58 std::move(file_reader
), EnvOptions(), kLength
));
61 PlainTableFileReader
reader(file_info
.get());
63 const uint32_t kReadSize
= 77;
64 for (uint32_t pos
= 0; pos
< kLength
; pos
+= kReadSize
) {
65 uint32_t read_size
= std::min(kLength
- pos
, kReadSize
);
67 ASSERT_TRUE(reader
.Read(pos
, read_size
, &out
));
68 ASSERT_EQ(0, out
.compare(tmp
.substr(pos
, read_size
)));
71 ASSERT_LT(uint32_t(string_source
->total_reads()), kLength
/ kReadSize
/ 2);
74 std::vector
<std::vector
<std::pair
<uint32_t, uint32_t>>> reads
= {
75 {{600, 30}, {590, 30}, {600, 20}, {600, 40}},
76 {{800, 20}, {100, 20}, {500, 20}, {1500, 20}, {100, 20}, {80, 20}},
77 {{1000, 20}, {500, 20}, {1000, 50}},
78 {{1000, 20}, {500, 20}, {500, 20}},
79 {{1000, 20}, {500, 20}, {200, 20}, {500, 20}},
80 {{1000, 20}, {500, 20}, {200, 20}, {1000, 50}},
81 {{600, 500}, {610, 20}, {100, 20}},
82 {{500, 100}, {490, 100}, {550, 50}},
85 std::vector
<int> num_file_reads
= {2, 6, 2, 2, 4, 3, 2, 2};
87 for (size_t i
= 0; i
< reads
.size(); i
++) {
88 string_source
->set_total_reads(0);
89 PlainTableFileReader
reader(file_info
.get());
90 for (auto p
: reads
[i
]) {
92 ASSERT_TRUE(reader
.Read(p
.first
, p
.second
, &out
));
93 ASSERT_EQ(0, out
.compare(tmp
.substr(p
.first
, p
.second
)));
95 ASSERT_EQ(num_file_reads
[i
], string_source
->total_reads());
99 class PlainTableDBTest
: public testing::Test
,
100 public testing::WithParamInterface
<bool> {
108 Options last_options_
;
111 PlainTableDBTest() : env_(Env::Default()) {}
113 ~PlainTableDBTest() {
115 EXPECT_OK(DestroyDB(dbname_
, Options()));
118 void SetUp() override
{
119 mmap_mode_
= GetParam();
120 dbname_
= test::TmpDir() + "/plain_table_db_test";
121 EXPECT_OK(DestroyDB(dbname_
, Options()));
126 // Return the current option configuration.
127 Options
CurrentOptions() {
130 PlainTableOptions plain_table_options
;
131 plain_table_options
.user_key_len
= 0;
132 plain_table_options
.bloom_bits_per_key
= 2;
133 plain_table_options
.hash_table_ratio
= 0.8;
134 plain_table_options
.index_sparseness
= 3;
135 plain_table_options
.huge_page_tlb_size
= 0;
136 plain_table_options
.encoding_type
= kPrefix
;
137 plain_table_options
.full_scan_mode
= false;
138 plain_table_options
.store_index_in_file
= false;
140 options
.table_factory
.reset(NewPlainTableFactory(plain_table_options
));
141 options
.memtable_factory
.reset(NewHashLinkListRepFactory(4, 0, 3, true));
143 options
.prefix_extractor
.reset(NewFixedPrefixTransform(8));
144 options
.allow_mmap_reads
= mmap_mode_
;
145 options
.allow_concurrent_memtable_write
= false;
150 return reinterpret_cast<DBImpl
*>(db_
);
153 void Reopen(Options
* options
= nullptr) {
154 ASSERT_OK(TryReopen(options
));
162 void DestroyAndReopen(Options
* options
= nullptr) {
163 //Destroy using last options
164 Destroy(&last_options_
);
165 ASSERT_OK(TryReopen(options
));
168 void Destroy(Options
* options
) {
171 ASSERT_OK(DestroyDB(dbname_
, *options
));
174 Status
PureReopen(Options
* options
, DB
** db
) {
175 return DB::Open(*options
, dbname_
, db
);
178 Status
TryReopen(Options
* options
= nullptr) {
182 if (options
!= nullptr) {
185 opts
= CurrentOptions();
186 opts
.create_if_missing
= true;
188 last_options_
= opts
;
190 return DB::Open(opts
, dbname_
, &db_
);
193 Status
Put(const Slice
& k
, const Slice
& v
) {
194 return db_
->Put(WriteOptions(), k
, v
);
197 Status
Delete(const std::string
& k
) {
198 return db_
->Delete(WriteOptions(), k
);
201 std::string
Get(const std::string
& k
, const Snapshot
* snapshot
= nullptr) {
203 options
.snapshot
= snapshot
;
205 Status s
= db_
->Get(options
, k
, &result
);
206 if (s
.IsNotFound()) {
207 result
= "NOT_FOUND";
208 } else if (!s
.ok()) {
209 result
= s
.ToString();
215 int NumTableFilesAtLevel(int level
) {
216 std::string property
;
217 EXPECT_TRUE(db_
->GetProperty(
218 "rocksdb.num-files-at-level" + NumberToString(level
), &property
));
219 return atoi(property
.c_str());
222 // Return spread of files per level
223 std::string
FilesPerLevel() {
225 size_t last_non_zero_offset
= 0;
226 for (int level
= 0; level
< db_
->NumberLevels(); level
++) {
227 int f
= NumTableFilesAtLevel(level
);
229 snprintf(buf
, sizeof(buf
), "%s%d", (level
? "," : ""), f
);
232 last_non_zero_offset
= result
.size();
235 result
.resize(last_non_zero_offset
);
239 std::string
IterStatus(Iterator
* iter
) {
242 result
= iter
->key().ToString() + "->" + iter
->value().ToString();
244 result
= "(invalid)";
250 TEST_P(PlainTableDBTest
, Empty
) {
251 ASSERT_TRUE(dbfull() != nullptr);
252 ASSERT_EQ("NOT_FOUND", Get("0000000000000foo"));
255 extern const uint64_t kPlainTableMagicNumber
;
257 class TestPlainTableReader
: public PlainTableReader
{
259 TestPlainTableReader(const EnvOptions
& env_options
,
260 const InternalKeyComparator
& icomparator
,
261 EncodingType encoding_type
, uint64_t file_size
,
262 int bloom_bits_per_key
, double hash_table_ratio
,
263 size_t index_sparseness
,
264 const TableProperties
* table_properties
,
265 unique_ptr
<RandomAccessFileReader
>&& file
,
266 const ImmutableCFOptions
& ioptions
,
267 bool* expect_bloom_not_match
, bool store_index_in_file
,
268 uint32_t column_family_id
,
269 const std::string
& column_family_name
)
270 : PlainTableReader(ioptions
, std::move(file
), env_options
, icomparator
,
271 encoding_type
, file_size
, table_properties
),
272 expect_bloom_not_match_(expect_bloom_not_match
) {
273 Status s
= MmapDataIfNeeded();
276 s
= PopulateIndex(const_cast<TableProperties
*>(table_properties
),
277 bloom_bits_per_key
, hash_table_ratio
, index_sparseness
,
281 TableProperties
* props
= const_cast<TableProperties
*>(table_properties
);
282 EXPECT_EQ(column_family_id
, static_cast<uint32_t>(props
->column_family_id
));
283 EXPECT_EQ(column_family_name
, props
->column_family_name
);
284 if (store_index_in_file
) {
285 auto bloom_version_ptr
= props
->user_collected_properties
.find(
286 PlainTablePropertyNames::kBloomVersion
);
287 EXPECT_TRUE(bloom_version_ptr
!= props
->user_collected_properties
.end());
288 EXPECT_EQ(bloom_version_ptr
->second
, std::string("1"));
289 if (ioptions
.bloom_locality
> 0) {
290 auto num_blocks_ptr
= props
->user_collected_properties
.find(
291 PlainTablePropertyNames::kNumBloomBlocks
);
292 EXPECT_TRUE(num_blocks_ptr
!= props
->user_collected_properties
.end());
297 virtual ~TestPlainTableReader() {}
300 virtual bool MatchBloom(uint32_t hash
) const override
{
301 bool ret
= PlainTableReader::MatchBloom(hash
);
302 if (*expect_bloom_not_match_
) {
309 bool* expect_bloom_not_match_
;
312 extern const uint64_t kPlainTableMagicNumber
;
313 class TestPlainTableFactory
: public PlainTableFactory
{
315 explicit TestPlainTableFactory(bool* expect_bloom_not_match
,
316 const PlainTableOptions
& options
,
317 uint32_t column_family_id
,
318 std::string column_family_name
)
319 : PlainTableFactory(options
),
320 bloom_bits_per_key_(options
.bloom_bits_per_key
),
321 hash_table_ratio_(options
.hash_table_ratio
),
322 index_sparseness_(options
.index_sparseness
),
323 store_index_in_file_(options
.store_index_in_file
),
324 expect_bloom_not_match_(expect_bloom_not_match
),
325 column_family_id_(column_family_id
),
326 column_family_name_(std::move(column_family_name
)) {}
328 Status
NewTableReader(
329 const TableReaderOptions
& table_reader_options
,
330 unique_ptr
<RandomAccessFileReader
>&& file
, uint64_t file_size
,
331 unique_ptr
<TableReader
>* table
,
332 bool prefetch_index_and_filter_in_cache
) const override
{
333 TableProperties
* props
= nullptr;
335 ReadTableProperties(file
.get(), file_size
, kPlainTableMagicNumber
,
336 table_reader_options
.ioptions
, &props
);
339 if (store_index_in_file_
) {
340 BlockHandle bloom_block_handle
;
341 s
= FindMetaBlock(file
.get(), file_size
, kPlainTableMagicNumber
,
342 table_reader_options
.ioptions
,
343 BloomBlockBuilder::kBloomBlock
, &bloom_block_handle
);
346 BlockHandle index_block_handle
;
347 s
= FindMetaBlock(file
.get(), file_size
, kPlainTableMagicNumber
,
348 table_reader_options
.ioptions
,
349 PlainTableIndexBuilder::kPlainTableIndexBlock
,
350 &index_block_handle
);
354 auto& user_props
= props
->user_collected_properties
;
355 auto encoding_type_prop
=
356 user_props
.find(PlainTablePropertyNames::kEncodingType
);
357 assert(encoding_type_prop
!= user_props
.end());
358 EncodingType encoding_type
= static_cast<EncodingType
>(
359 DecodeFixed32(encoding_type_prop
->second
.c_str()));
361 std::unique_ptr
<PlainTableReader
> new_reader(new TestPlainTableReader(
362 table_reader_options
.env_options
,
363 table_reader_options
.internal_comparator
, encoding_type
, file_size
,
364 bloom_bits_per_key_
, hash_table_ratio_
, index_sparseness_
, props
,
365 std::move(file
), table_reader_options
.ioptions
, expect_bloom_not_match_
,
366 store_index_in_file_
, column_family_id_
, column_family_name_
));
368 *table
= std::move(new_reader
);
373 int bloom_bits_per_key_
;
374 double hash_table_ratio_
;
375 size_t index_sparseness_
;
376 bool store_index_in_file_
;
377 bool* expect_bloom_not_match_
;
378 const uint32_t column_family_id_
;
379 const std::string column_family_name_
;
382 TEST_P(PlainTableDBTest
, Flush
) {
383 for (size_t huge_page_tlb_size
= 0; huge_page_tlb_size
<= 2 * 1024 * 1024;
384 huge_page_tlb_size
+= 2 * 1024 * 1024) {
385 for (EncodingType encoding_type
: {kPlain
, kPrefix
}) {
386 for (int bloom_bits
= 0; bloom_bits
<= 117; bloom_bits
+= 117) {
387 for (int total_order
= 0; total_order
<= 1; total_order
++) {
388 for (int store_index_in_file
= 0; store_index_in_file
<= 1;
389 ++store_index_in_file
) {
390 Options options
= CurrentOptions();
391 options
.create_if_missing
= true;
392 // Set only one bucket to force bucket conflict.
393 // Test index interval for the same prefix to be 1, 2 and 4
395 options
.prefix_extractor
.reset();
397 PlainTableOptions plain_table_options
;
398 plain_table_options
.user_key_len
= 0;
399 plain_table_options
.bloom_bits_per_key
= bloom_bits
;
400 plain_table_options
.hash_table_ratio
= 0;
401 plain_table_options
.index_sparseness
= 2;
402 plain_table_options
.huge_page_tlb_size
= huge_page_tlb_size
;
403 plain_table_options
.encoding_type
= encoding_type
;
404 plain_table_options
.full_scan_mode
= false;
405 plain_table_options
.store_index_in_file
= store_index_in_file
;
407 options
.table_factory
.reset(
408 NewPlainTableFactory(plain_table_options
));
410 PlainTableOptions plain_table_options
;
411 plain_table_options
.user_key_len
= 0;
412 plain_table_options
.bloom_bits_per_key
= bloom_bits
;
413 plain_table_options
.hash_table_ratio
= 0.75;
414 plain_table_options
.index_sparseness
= 16;
415 plain_table_options
.huge_page_tlb_size
= huge_page_tlb_size
;
416 plain_table_options
.encoding_type
= encoding_type
;
417 plain_table_options
.full_scan_mode
= false;
418 plain_table_options
.store_index_in_file
= store_index_in_file
;
420 options
.table_factory
.reset(
421 NewPlainTableFactory(plain_table_options
));
423 DestroyAndReopen(&options
);
425 ASSERT_TRUE(dbfull()->GetIntProperty(
426 "rocksdb.estimate-table-readers-mem", &int_num
));
427 ASSERT_EQ(int_num
, 0U);
429 ASSERT_OK(Put("1000000000000foo", "v1"));
430 ASSERT_OK(Put("0000000000000bar", "v2"));
431 ASSERT_OK(Put("1000000000000foo", "v3"));
432 dbfull()->TEST_FlushMemTable();
434 ASSERT_TRUE(dbfull()->GetIntProperty(
435 "rocksdb.estimate-table-readers-mem", &int_num
));
436 ASSERT_GT(int_num
, 0U);
438 TablePropertiesCollection ptc
;
439 reinterpret_cast<DB
*>(dbfull())->GetPropertiesOfAllTables(&ptc
);
440 ASSERT_EQ(1U, ptc
.size());
441 auto row
= ptc
.begin();
442 auto tp
= row
->second
;
444 if (!store_index_in_file
) {
445 ASSERT_EQ(total_order
? "4" : "12",
446 (tp
->user_collected_properties
)
447 .at("plain_table_hash_table_size"));
448 ASSERT_EQ("0", (tp
->user_collected_properties
)
449 .at("plain_table_sub_index_size"));
451 ASSERT_EQ("0", (tp
->user_collected_properties
)
452 .at("plain_table_hash_table_size"));
453 ASSERT_EQ("0", (tp
->user_collected_properties
)
454 .at("plain_table_sub_index_size"));
456 ASSERT_EQ("v3", Get("1000000000000foo"));
457 ASSERT_EQ("v2", Get("0000000000000bar"));
465 TEST_P(PlainTableDBTest
, Flush2
) {
466 for (size_t huge_page_tlb_size
= 0; huge_page_tlb_size
<= 2 * 1024 * 1024;
467 huge_page_tlb_size
+= 2 * 1024 * 1024) {
468 for (EncodingType encoding_type
: {kPlain
, kPrefix
}) {
469 for (int bloom_bits
= 0; bloom_bits
<= 117; bloom_bits
+= 117) {
470 for (int total_order
= 0; total_order
<= 1; total_order
++) {
471 for (int store_index_in_file
= 0; store_index_in_file
<= 1;
472 ++store_index_in_file
) {
473 if (encoding_type
== kPrefix
&& total_order
) {
476 if (!bloom_bits
&& store_index_in_file
) {
479 if (total_order
&& store_index_in_file
) {
482 bool expect_bloom_not_match
= false;
483 Options options
= CurrentOptions();
484 options
.create_if_missing
= true;
485 // Set only one bucket to force bucket conflict.
486 // Test index interval for the same prefix to be 1, 2 and 4
487 PlainTableOptions plain_table_options
;
489 options
.prefix_extractor
= nullptr;
490 plain_table_options
.hash_table_ratio
= 0;
491 plain_table_options
.index_sparseness
= 2;
493 plain_table_options
.hash_table_ratio
= 0.75;
494 plain_table_options
.index_sparseness
= 16;
496 plain_table_options
.user_key_len
= kPlainTableVariableLength
;
497 plain_table_options
.bloom_bits_per_key
= bloom_bits
;
498 plain_table_options
.huge_page_tlb_size
= huge_page_tlb_size
;
499 plain_table_options
.encoding_type
= encoding_type
;
500 plain_table_options
.store_index_in_file
= store_index_in_file
;
501 options
.table_factory
.reset(new TestPlainTableFactory(
502 &expect_bloom_not_match
, plain_table_options
,
503 0 /* column_family_id */, kDefaultColumnFamilyName
));
505 DestroyAndReopen(&options
);
506 ASSERT_OK(Put("0000000000000bar", "b"));
507 ASSERT_OK(Put("1000000000000foo", "v1"));
508 dbfull()->TEST_FlushMemTable();
510 ASSERT_OK(Put("1000000000000foo", "v2"));
511 dbfull()->TEST_FlushMemTable();
512 ASSERT_EQ("v2", Get("1000000000000foo"));
514 ASSERT_OK(Put("0000000000000eee", "v3"));
515 dbfull()->TEST_FlushMemTable();
516 ASSERT_EQ("v3", Get("0000000000000eee"));
518 ASSERT_OK(Delete("0000000000000bar"));
519 dbfull()->TEST_FlushMemTable();
520 ASSERT_EQ("NOT_FOUND", Get("0000000000000bar"));
522 ASSERT_OK(Put("0000000000000eee", "v5"));
523 ASSERT_OK(Put("9000000000000eee", "v5"));
524 dbfull()->TEST_FlushMemTable();
525 ASSERT_EQ("v5", Get("0000000000000eee"));
528 if (bloom_bits
> 0) {
529 // Neither key nor value should exist.
530 expect_bloom_not_match
= true;
531 ASSERT_EQ("NOT_FOUND", Get("5_not00000000bar"));
532 // Key doesn't exist any more but prefix exists.
534 ASSERT_EQ("NOT_FOUND", Get("1000000000000not"));
535 ASSERT_EQ("NOT_FOUND", Get("0000000000000not"));
537 expect_bloom_not_match
= false;
546 TEST_P(PlainTableDBTest
, Iterator
) {
547 for (size_t huge_page_tlb_size
= 0; huge_page_tlb_size
<= 2 * 1024 * 1024;
548 huge_page_tlb_size
+= 2 * 1024 * 1024) {
549 for (EncodingType encoding_type
: {kPlain
, kPrefix
}) {
550 for (int bloom_bits
= 0; bloom_bits
<= 117; bloom_bits
+= 117) {
551 for (int total_order
= 0; total_order
<= 1; total_order
++) {
552 if (encoding_type
== kPrefix
&& total_order
== 1) {
555 bool expect_bloom_not_match
= false;
556 Options options
= CurrentOptions();
557 options
.create_if_missing
= true;
558 // Set only one bucket to force bucket conflict.
559 // Test index interval for the same prefix to be 1, 2 and 4
561 options
.prefix_extractor
= nullptr;
563 PlainTableOptions plain_table_options
;
564 plain_table_options
.user_key_len
= 16;
565 plain_table_options
.bloom_bits_per_key
= bloom_bits
;
566 plain_table_options
.hash_table_ratio
= 0;
567 plain_table_options
.index_sparseness
= 2;
568 plain_table_options
.huge_page_tlb_size
= huge_page_tlb_size
;
569 plain_table_options
.encoding_type
= encoding_type
;
571 options
.table_factory
.reset(new TestPlainTableFactory(
572 &expect_bloom_not_match
, plain_table_options
,
573 0 /* column_family_id */, kDefaultColumnFamilyName
));
575 PlainTableOptions plain_table_options
;
576 plain_table_options
.user_key_len
= 16;
577 plain_table_options
.bloom_bits_per_key
= bloom_bits
;
578 plain_table_options
.hash_table_ratio
= 0.75;
579 plain_table_options
.index_sparseness
= 16;
580 plain_table_options
.huge_page_tlb_size
= huge_page_tlb_size
;
581 plain_table_options
.encoding_type
= encoding_type
;
583 options
.table_factory
.reset(new TestPlainTableFactory(
584 &expect_bloom_not_match
, plain_table_options
,
585 0 /* column_family_id */, kDefaultColumnFamilyName
));
587 DestroyAndReopen(&options
);
589 ASSERT_OK(Put("1000000000foo002", "v_2"));
590 ASSERT_OK(Put("0000000000000bar", "random"));
591 ASSERT_OK(Put("1000000000foo001", "v1"));
592 ASSERT_OK(Put("3000000000000bar", "bar_v"));
593 ASSERT_OK(Put("1000000000foo003", "v__3"));
594 ASSERT_OK(Put("1000000000foo004", "v__4"));
595 ASSERT_OK(Put("1000000000foo005", "v__5"));
596 ASSERT_OK(Put("1000000000foo007", "v__7"));
597 ASSERT_OK(Put("1000000000foo008", "v__8"));
598 dbfull()->TEST_FlushMemTable();
599 ASSERT_EQ("v1", Get("1000000000foo001"));
600 ASSERT_EQ("v__3", Get("1000000000foo003"));
601 Iterator
* iter
= dbfull()->NewIterator(ReadOptions());
602 iter
->Seek("1000000000foo000");
603 ASSERT_TRUE(iter
->Valid());
604 ASSERT_EQ("1000000000foo001", iter
->key().ToString());
605 ASSERT_EQ("v1", iter
->value().ToString());
608 ASSERT_TRUE(iter
->Valid());
609 ASSERT_EQ("1000000000foo002", iter
->key().ToString());
610 ASSERT_EQ("v_2", iter
->value().ToString());
613 ASSERT_TRUE(iter
->Valid());
614 ASSERT_EQ("1000000000foo003", iter
->key().ToString());
615 ASSERT_EQ("v__3", iter
->value().ToString());
618 ASSERT_TRUE(iter
->Valid());
619 ASSERT_EQ("1000000000foo004", iter
->key().ToString());
620 ASSERT_EQ("v__4", iter
->value().ToString());
622 iter
->Seek("3000000000000bar");
623 ASSERT_TRUE(iter
->Valid());
624 ASSERT_EQ("3000000000000bar", iter
->key().ToString());
625 ASSERT_EQ("bar_v", iter
->value().ToString());
627 iter
->Seek("1000000000foo000");
628 ASSERT_TRUE(iter
->Valid());
629 ASSERT_EQ("1000000000foo001", iter
->key().ToString());
630 ASSERT_EQ("v1", iter
->value().ToString());
632 iter
->Seek("1000000000foo005");
633 ASSERT_TRUE(iter
->Valid());
634 ASSERT_EQ("1000000000foo005", iter
->key().ToString());
635 ASSERT_EQ("v__5", iter
->value().ToString());
637 iter
->Seek("1000000000foo006");
638 ASSERT_TRUE(iter
->Valid());
639 ASSERT_EQ("1000000000foo007", iter
->key().ToString());
640 ASSERT_EQ("v__7", iter
->value().ToString());
642 iter
->Seek("1000000000foo008");
643 ASSERT_TRUE(iter
->Valid());
644 ASSERT_EQ("1000000000foo008", iter
->key().ToString());
645 ASSERT_EQ("v__8", iter
->value().ToString());
647 if (total_order
== 0) {
648 iter
->Seek("1000000000foo009");
649 ASSERT_TRUE(iter
->Valid());
650 ASSERT_EQ("3000000000000bar", iter
->key().ToString());
654 if (bloom_bits
> 0) {
656 // Neither key nor value should exist.
657 expect_bloom_not_match
= true;
658 iter
->Seek("2not000000000bar");
659 ASSERT_TRUE(!iter
->Valid());
660 ASSERT_EQ("NOT_FOUND", Get("2not000000000bar"));
661 expect_bloom_not_match
= false;
663 expect_bloom_not_match
= true;
664 ASSERT_EQ("NOT_FOUND", Get("2not000000000bar"));
665 expect_bloom_not_match
= false;
677 std::string
MakeLongKey(size_t length
, char c
) {
678 return std::string(length
, c
);
682 TEST_P(PlainTableDBTest
, IteratorLargeKeys
) {
683 Options options
= CurrentOptions();
685 PlainTableOptions plain_table_options
;
686 plain_table_options
.user_key_len
= 0;
687 plain_table_options
.bloom_bits_per_key
= 0;
688 plain_table_options
.hash_table_ratio
= 0;
690 options
.table_factory
.reset(NewPlainTableFactory(plain_table_options
));
691 options
.create_if_missing
= true;
692 options
.prefix_extractor
.reset();
693 DestroyAndReopen(&options
);
695 std::string key_list
[] = {
696 MakeLongKey(30, '0'),
697 MakeLongKey(16, '1'),
698 MakeLongKey(32, '2'),
699 MakeLongKey(60, '3'),
700 MakeLongKey(90, '4'),
701 MakeLongKey(50, '5'),
705 for (size_t i
= 0; i
< 7; i
++) {
706 ASSERT_OK(Put(key_list
[i
], ToString(i
)));
709 dbfull()->TEST_FlushMemTable();
711 Iterator
* iter
= dbfull()->NewIterator(ReadOptions());
712 iter
->Seek(key_list
[0]);
714 for (size_t i
= 0; i
< 7; i
++) {
715 ASSERT_TRUE(iter
->Valid());
716 ASSERT_EQ(key_list
[i
], iter
->key().ToString());
717 ASSERT_EQ(ToString(i
), iter
->value().ToString());
721 ASSERT_TRUE(!iter
->Valid());
727 std::string
MakeLongKeyWithPrefix(size_t length
, char c
) {
728 return "00000000" + std::string(length
- 8, c
);
732 TEST_P(PlainTableDBTest
, IteratorLargeKeysWithPrefix
) {
733 Options options
= CurrentOptions();
735 PlainTableOptions plain_table_options
;
736 plain_table_options
.user_key_len
= 16;
737 plain_table_options
.bloom_bits_per_key
= 0;
738 plain_table_options
.hash_table_ratio
= 0.8;
739 plain_table_options
.index_sparseness
= 3;
740 plain_table_options
.huge_page_tlb_size
= 0;
741 plain_table_options
.encoding_type
= kPrefix
;
743 options
.table_factory
.reset(NewPlainTableFactory(plain_table_options
));
744 options
.create_if_missing
= true;
745 DestroyAndReopen(&options
);
747 std::string key_list
[] = {
748 MakeLongKeyWithPrefix(30, '0'), MakeLongKeyWithPrefix(16, '1'),
749 MakeLongKeyWithPrefix(32, '2'), MakeLongKeyWithPrefix(60, '3'),
750 MakeLongKeyWithPrefix(90, '4'), MakeLongKeyWithPrefix(50, '5'),
751 MakeLongKeyWithPrefix(26, '6')};
753 for (size_t i
= 0; i
< 7; i
++) {
754 ASSERT_OK(Put(key_list
[i
], ToString(i
)));
757 dbfull()->TEST_FlushMemTable();
759 Iterator
* iter
= dbfull()->NewIterator(ReadOptions());
760 iter
->Seek(key_list
[0]);
762 for (size_t i
= 0; i
< 7; i
++) {
763 ASSERT_TRUE(iter
->Valid());
764 ASSERT_EQ(key_list
[i
], iter
->key().ToString());
765 ASSERT_EQ(ToString(i
), iter
->value().ToString());
769 ASSERT_TRUE(!iter
->Valid());
774 TEST_P(PlainTableDBTest
, IteratorReverseSuffixComparator
) {
775 Options options
= CurrentOptions();
776 options
.create_if_missing
= true;
777 // Set only one bucket to force bucket conflict.
778 // Test index interval for the same prefix to be 1, 2 and 4
779 test::SimpleSuffixReverseComparator comp
;
780 options
.comparator
= &comp
;
781 DestroyAndReopen(&options
);
783 ASSERT_OK(Put("1000000000foo002", "v_2"));
784 ASSERT_OK(Put("0000000000000bar", "random"));
785 ASSERT_OK(Put("1000000000foo001", "v1"));
786 ASSERT_OK(Put("3000000000000bar", "bar_v"));
787 ASSERT_OK(Put("1000000000foo003", "v__3"));
788 ASSERT_OK(Put("1000000000foo004", "v__4"));
789 ASSERT_OK(Put("1000000000foo005", "v__5"));
790 ASSERT_OK(Put("1000000000foo007", "v__7"));
791 ASSERT_OK(Put("1000000000foo008", "v__8"));
792 dbfull()->TEST_FlushMemTable();
793 ASSERT_EQ("v1", Get("1000000000foo001"));
794 ASSERT_EQ("v__3", Get("1000000000foo003"));
795 Iterator
* iter
= dbfull()->NewIterator(ReadOptions());
796 iter
->Seek("1000000000foo009");
797 ASSERT_TRUE(iter
->Valid());
798 ASSERT_EQ("1000000000foo008", iter
->key().ToString());
799 ASSERT_EQ("v__8", iter
->value().ToString());
802 ASSERT_TRUE(iter
->Valid());
803 ASSERT_EQ("1000000000foo007", iter
->key().ToString());
804 ASSERT_EQ("v__7", iter
->value().ToString());
807 ASSERT_TRUE(iter
->Valid());
808 ASSERT_EQ("1000000000foo005", iter
->key().ToString());
809 ASSERT_EQ("v__5", iter
->value().ToString());
812 ASSERT_TRUE(iter
->Valid());
813 ASSERT_EQ("1000000000foo004", iter
->key().ToString());
814 ASSERT_EQ("v__4", iter
->value().ToString());
816 iter
->Seek("3000000000000bar");
817 ASSERT_TRUE(iter
->Valid());
818 ASSERT_EQ("3000000000000bar", iter
->key().ToString());
819 ASSERT_EQ("bar_v", iter
->value().ToString());
821 iter
->Seek("1000000000foo005");
822 ASSERT_TRUE(iter
->Valid());
823 ASSERT_EQ("1000000000foo005", iter
->key().ToString());
824 ASSERT_EQ("v__5", iter
->value().ToString());
826 iter
->Seek("1000000000foo006");
827 ASSERT_TRUE(iter
->Valid());
828 ASSERT_EQ("1000000000foo005", iter
->key().ToString());
829 ASSERT_EQ("v__5", iter
->value().ToString());
831 iter
->Seek("1000000000foo008");
832 ASSERT_TRUE(iter
->Valid());
833 ASSERT_EQ("1000000000foo008", iter
->key().ToString());
834 ASSERT_EQ("v__8", iter
->value().ToString());
836 iter
->Seek("1000000000foo000");
837 ASSERT_TRUE(iter
->Valid());
838 ASSERT_EQ("3000000000000bar", iter
->key().ToString());
843 TEST_P(PlainTableDBTest
, HashBucketConflict
) {
844 for (size_t huge_page_tlb_size
= 0; huge_page_tlb_size
<= 2 * 1024 * 1024;
845 huge_page_tlb_size
+= 2 * 1024 * 1024) {
846 for (unsigned char i
= 1; i
<= 3; i
++) {
847 Options options
= CurrentOptions();
848 options
.create_if_missing
= true;
849 // Set only one bucket to force bucket conflict.
850 // Test index interval for the same prefix to be 1, 2 and 4
852 PlainTableOptions plain_table_options
;
853 plain_table_options
.user_key_len
= 16;
854 plain_table_options
.bloom_bits_per_key
= 0;
855 plain_table_options
.hash_table_ratio
= 0;
856 plain_table_options
.index_sparseness
= 2 ^ i
;
857 plain_table_options
.huge_page_tlb_size
= huge_page_tlb_size
;
859 options
.table_factory
.reset(NewPlainTableFactory(plain_table_options
));
861 DestroyAndReopen(&options
);
862 ASSERT_OK(Put("5000000000000fo0", "v1"));
863 ASSERT_OK(Put("5000000000000fo1", "v2"));
864 ASSERT_OK(Put("5000000000000fo2", "v"));
865 ASSERT_OK(Put("2000000000000fo0", "v3"));
866 ASSERT_OK(Put("2000000000000fo1", "v4"));
867 ASSERT_OK(Put("2000000000000fo2", "v"));
868 ASSERT_OK(Put("2000000000000fo3", "v"));
870 dbfull()->TEST_FlushMemTable();
872 ASSERT_EQ("v1", Get("5000000000000fo0"));
873 ASSERT_EQ("v2", Get("5000000000000fo1"));
874 ASSERT_EQ("v3", Get("2000000000000fo0"));
875 ASSERT_EQ("v4", Get("2000000000000fo1"));
877 ASSERT_EQ("NOT_FOUND", Get("5000000000000bar"));
878 ASSERT_EQ("NOT_FOUND", Get("2000000000000bar"));
879 ASSERT_EQ("NOT_FOUND", Get("5000000000000fo8"));
880 ASSERT_EQ("NOT_FOUND", Get("2000000000000fo8"));
883 Iterator
* iter
= dbfull()->NewIterator(ro
);
885 iter
->Seek("5000000000000fo0");
886 ASSERT_TRUE(iter
->Valid());
887 ASSERT_EQ("5000000000000fo0", iter
->key().ToString());
889 ASSERT_TRUE(iter
->Valid());
890 ASSERT_EQ("5000000000000fo1", iter
->key().ToString());
892 iter
->Seek("5000000000000fo1");
893 ASSERT_TRUE(iter
->Valid());
894 ASSERT_EQ("5000000000000fo1", iter
->key().ToString());
896 iter
->Seek("2000000000000fo0");
897 ASSERT_TRUE(iter
->Valid());
898 ASSERT_EQ("2000000000000fo0", iter
->key().ToString());
900 ASSERT_TRUE(iter
->Valid());
901 ASSERT_EQ("2000000000000fo1", iter
->key().ToString());
903 iter
->Seek("2000000000000fo1");
904 ASSERT_TRUE(iter
->Valid());
905 ASSERT_EQ("2000000000000fo1", iter
->key().ToString());
907 iter
->Seek("2000000000000bar");
908 ASSERT_TRUE(iter
->Valid());
909 ASSERT_EQ("2000000000000fo0", iter
->key().ToString());
911 iter
->Seek("5000000000000bar");
912 ASSERT_TRUE(iter
->Valid());
913 ASSERT_EQ("5000000000000fo0", iter
->key().ToString());
915 iter
->Seek("2000000000000fo8");
916 ASSERT_TRUE(!iter
->Valid() ||
917 options
.comparator
->Compare(iter
->key(), "20000001") > 0);
919 iter
->Seek("5000000000000fo8");
920 ASSERT_TRUE(!iter
->Valid());
922 iter
->Seek("1000000000000fo2");
923 ASSERT_TRUE(!iter
->Valid());
925 iter
->Seek("3000000000000fo2");
926 ASSERT_TRUE(!iter
->Valid());
928 iter
->Seek("8000000000000fo2");
929 ASSERT_TRUE(!iter
->Valid());
936 TEST_P(PlainTableDBTest
, HashBucketConflictReverseSuffixComparator
) {
937 for (size_t huge_page_tlb_size
= 0; huge_page_tlb_size
<= 2 * 1024 * 1024;
938 huge_page_tlb_size
+= 2 * 1024 * 1024) {
939 for (unsigned char i
= 1; i
<= 3; i
++) {
940 Options options
= CurrentOptions();
941 options
.create_if_missing
= true;
942 test::SimpleSuffixReverseComparator comp
;
943 options
.comparator
= &comp
;
944 // Set only one bucket to force bucket conflict.
945 // Test index interval for the same prefix to be 1, 2 and 4
947 PlainTableOptions plain_table_options
;
948 plain_table_options
.user_key_len
= 16;
949 plain_table_options
.bloom_bits_per_key
= 0;
950 plain_table_options
.hash_table_ratio
= 0;
951 plain_table_options
.index_sparseness
= 2 ^ i
;
952 plain_table_options
.huge_page_tlb_size
= huge_page_tlb_size
;
954 options
.table_factory
.reset(NewPlainTableFactory(plain_table_options
));
955 DestroyAndReopen(&options
);
956 ASSERT_OK(Put("5000000000000fo0", "v1"));
957 ASSERT_OK(Put("5000000000000fo1", "v2"));
958 ASSERT_OK(Put("5000000000000fo2", "v"));
959 ASSERT_OK(Put("2000000000000fo0", "v3"));
960 ASSERT_OK(Put("2000000000000fo1", "v4"));
961 ASSERT_OK(Put("2000000000000fo2", "v"));
962 ASSERT_OK(Put("2000000000000fo3", "v"));
964 dbfull()->TEST_FlushMemTable();
966 ASSERT_EQ("v1", Get("5000000000000fo0"));
967 ASSERT_EQ("v2", Get("5000000000000fo1"));
968 ASSERT_EQ("v3", Get("2000000000000fo0"));
969 ASSERT_EQ("v4", Get("2000000000000fo1"));
971 ASSERT_EQ("NOT_FOUND", Get("5000000000000bar"));
972 ASSERT_EQ("NOT_FOUND", Get("2000000000000bar"));
973 ASSERT_EQ("NOT_FOUND", Get("5000000000000fo8"));
974 ASSERT_EQ("NOT_FOUND", Get("2000000000000fo8"));
977 Iterator
* iter
= dbfull()->NewIterator(ro
);
979 iter
->Seek("5000000000000fo1");
980 ASSERT_TRUE(iter
->Valid());
981 ASSERT_EQ("5000000000000fo1", iter
->key().ToString());
983 ASSERT_TRUE(iter
->Valid());
984 ASSERT_EQ("5000000000000fo0", iter
->key().ToString());
986 iter
->Seek("5000000000000fo1");
987 ASSERT_TRUE(iter
->Valid());
988 ASSERT_EQ("5000000000000fo1", iter
->key().ToString());
990 iter
->Seek("2000000000000fo1");
991 ASSERT_TRUE(iter
->Valid());
992 ASSERT_EQ("2000000000000fo1", iter
->key().ToString());
994 ASSERT_TRUE(iter
->Valid());
995 ASSERT_EQ("2000000000000fo0", iter
->key().ToString());
997 iter
->Seek("2000000000000fo1");
998 ASSERT_TRUE(iter
->Valid());
999 ASSERT_EQ("2000000000000fo1", iter
->key().ToString());
1001 iter
->Seek("2000000000000var");
1002 ASSERT_TRUE(iter
->Valid());
1003 ASSERT_EQ("2000000000000fo3", iter
->key().ToString());
1005 iter
->Seek("5000000000000var");
1006 ASSERT_TRUE(iter
->Valid());
1007 ASSERT_EQ("5000000000000fo2", iter
->key().ToString());
1009 std::string seek_key
= "2000000000000bar";
1010 iter
->Seek(seek_key
);
1011 ASSERT_TRUE(!iter
->Valid() ||
1012 options
.prefix_extractor
->Transform(iter
->key()) !=
1013 options
.prefix_extractor
->Transform(seek_key
));
1015 iter
->Seek("1000000000000fo2");
1016 ASSERT_TRUE(!iter
->Valid());
1018 iter
->Seek("3000000000000fo2");
1019 ASSERT_TRUE(!iter
->Valid());
1021 iter
->Seek("8000000000000fo2");
1022 ASSERT_TRUE(!iter
->Valid());
1029 TEST_P(PlainTableDBTest
, NonExistingKeyToNonEmptyBucket
) {
1030 Options options
= CurrentOptions();
1031 options
.create_if_missing
= true;
1032 // Set only one bucket to force bucket conflict.
1033 // Test index interval for the same prefix to be 1, 2 and 4
1034 PlainTableOptions plain_table_options
;
1035 plain_table_options
.user_key_len
= 16;
1036 plain_table_options
.bloom_bits_per_key
= 0;
1037 plain_table_options
.hash_table_ratio
= 0;
1038 plain_table_options
.index_sparseness
= 5;
1040 options
.table_factory
.reset(NewPlainTableFactory(plain_table_options
));
1041 DestroyAndReopen(&options
);
1042 ASSERT_OK(Put("5000000000000fo0", "v1"));
1043 ASSERT_OK(Put("5000000000000fo1", "v2"));
1044 ASSERT_OK(Put("5000000000000fo2", "v3"));
1046 dbfull()->TEST_FlushMemTable();
1048 ASSERT_EQ("v1", Get("5000000000000fo0"));
1049 ASSERT_EQ("v2", Get("5000000000000fo1"));
1050 ASSERT_EQ("v3", Get("5000000000000fo2"));
1052 ASSERT_EQ("NOT_FOUND", Get("8000000000000bar"));
1053 ASSERT_EQ("NOT_FOUND", Get("1000000000000bar"));
1055 Iterator
* iter
= dbfull()->NewIterator(ReadOptions());
1057 iter
->Seek("5000000000000bar");
1058 ASSERT_TRUE(iter
->Valid());
1059 ASSERT_EQ("5000000000000fo0", iter
->key().ToString());
1061 iter
->Seek("5000000000000fo8");
1062 ASSERT_TRUE(!iter
->Valid());
1064 iter
->Seek("1000000000000fo2");
1065 ASSERT_TRUE(!iter
->Valid());
1067 iter
->Seek("8000000000000fo2");
1068 ASSERT_TRUE(!iter
->Valid());
1073 static std::string
Key(int i
) {
1075 snprintf(buf
, sizeof(buf
), "key_______%06d", i
);
1076 return std::string(buf
);
1079 static std::string
RandomString(Random
* rnd
, int len
) {
1081 test::RandomString(rnd
, len
, &r
);
1085 TEST_P(PlainTableDBTest
, CompactionTrigger
) {
1086 Options options
= CurrentOptions();
1087 options
.write_buffer_size
= 120 << 10; // 100KB
1088 options
.num_levels
= 3;
1089 options
.level0_file_num_compaction_trigger
= 3;
1094 for (int num
= 0; num
< options
.level0_file_num_compaction_trigger
- 1;
1096 std::vector
<std::string
> values
;
1097 // Write 120KB (10 values, each 12K)
1098 for (int i
= 0; i
< 10; i
++) {
1099 values
.push_back(RandomString(&rnd
, 12000));
1100 ASSERT_OK(Put(Key(i
), values
[i
]));
1102 ASSERT_OK(Put(Key(999), ""));
1103 dbfull()->TEST_WaitForFlushMemTable();
1104 ASSERT_EQ(NumTableFilesAtLevel(0), num
+ 1);
1107 //generate one more file in level-0, and should trigger level-0 compaction
1108 std::vector
<std::string
> values
;
1109 for (int i
= 0; i
< 12; i
++) {
1110 values
.push_back(RandomString(&rnd
, 10000));
1111 ASSERT_OK(Put(Key(i
), values
[i
]));
1113 ASSERT_OK(Put(Key(999), ""));
1114 dbfull()->TEST_WaitForCompact();
1116 ASSERT_EQ(NumTableFilesAtLevel(0), 0);
1117 ASSERT_EQ(NumTableFilesAtLevel(1), 1);
1120 TEST_P(PlainTableDBTest
, AdaptiveTable
) {
1121 Options options
= CurrentOptions();
1122 options
.create_if_missing
= true;
1124 options
.table_factory
.reset(NewPlainTableFactory());
1125 DestroyAndReopen(&options
);
1127 ASSERT_OK(Put("1000000000000foo", "v1"));
1128 ASSERT_OK(Put("0000000000000bar", "v2"));
1129 ASSERT_OK(Put("1000000000000foo", "v3"));
1130 dbfull()->TEST_FlushMemTable();
1132 options
.create_if_missing
= false;
1133 std::shared_ptr
<TableFactory
> dummy_factory
;
1134 std::shared_ptr
<TableFactory
> block_based_factory(
1135 NewBlockBasedTableFactory());
1136 options
.table_factory
.reset(NewAdaptiveTableFactory(
1137 block_based_factory
, dummy_factory
, dummy_factory
));
1139 ASSERT_EQ("v3", Get("1000000000000foo"));
1140 ASSERT_EQ("v2", Get("0000000000000bar"));
1142 ASSERT_OK(Put("2000000000000foo", "v4"));
1143 ASSERT_OK(Put("3000000000000bar", "v5"));
1144 dbfull()->TEST_FlushMemTable();
1145 ASSERT_EQ("v4", Get("2000000000000foo"));
1146 ASSERT_EQ("v5", Get("3000000000000bar"));
1149 ASSERT_EQ("v3", Get("1000000000000foo"));
1150 ASSERT_EQ("v2", Get("0000000000000bar"));
1151 ASSERT_EQ("v4", Get("2000000000000foo"));
1152 ASSERT_EQ("v5", Get("3000000000000bar"));
1154 options
.table_factory
.reset(NewBlockBasedTableFactory());
1156 ASSERT_NE("v3", Get("1000000000000foo"));
1158 options
.table_factory
.reset(NewPlainTableFactory());
1160 ASSERT_NE("v5", Get("3000000000000bar"));
1163 INSTANTIATE_TEST_CASE_P(PlainTableDBTest
, PlainTableDBTest
, ::testing::Bool());
1165 } // namespace rocksdb
1167 int main(int argc
, char** argv
) {
1168 ::testing::InitGoogleTest(&argc
, argv
);
1169 return RUN_ALL_TESTS();
1175 int main(int argc
, char** argv
) {
1176 fprintf(stderr
, "SKIPPED as plain table is not supported in ROCKSDB_LITE\n");
1180 #endif // !ROCKSDB_LITE