[ceph.git] / ceph / src / rocksdb / db / db_merge_operand_test.cc

//  Copyright (c) 2018-present, Facebook, Inc.  All rights reserved.
//  This source code is licensed under both the GPLv2 (found in the
//  COPYING file in the root directory) and Apache 2.0 License
//  (found in the LICENSE.Apache file in the root directory).

#include "db/db_test_util.h"
#include "port/stack_trace.h"
#include "rocksdb/perf_context.h"
#include "rocksdb/utilities/debug.h"
#include "table/block_based/block_builder.h"
#if !defined(ROCKSDB_LITE)
#include "test_util/sync_point.h"
#endif
#include "rocksdb/merge_operator.h"
#include "utilities/fault_injection_env.h"
#include "utilities/merge_operators.h"
#include "utilities/merge_operators/sortlist.h"
#include "utilities/merge_operators/string_append/stringappend2.h"

namespace ROCKSDB_NAMESPACE {

class DBMergeOperandTest : public DBTestBase {
 public:
  DBMergeOperandTest()
      : DBTestBase("/db_merge_operand_test", /*env_do_fsync=*/true) {}
};

TEST_F(DBMergeOperandTest, GetMergeOperandsBasic) {
  class LimitedStringAppendMergeOp : public StringAppendTESTOperator {
   public:
    LimitedStringAppendMergeOp(int limit, char delim)
        : StringAppendTESTOperator(delim), limit_(limit) {}

    const char* Name() const override {
      return "DBMergeOperatorTest::LimitedStringAppendMergeOp";
    }

    bool ShouldMerge(const std::vector<Slice>& operands) const override {
      if (operands.size() > 0 && limit_ > 0 && operands.size() >= limit_) {
        return true;
      }
      return false;
    }

   private:
    size_t limit_ = 0;
  };

  Options options;
  options.create_if_missing = true;
  // Use only the latest two merge operands.
  options.merge_operator = std::make_shared<LimitedStringAppendMergeOp>(2, ',');
  options.env = env_;
  Reopen(options);
  int num_records = 4;
  int number_of_operands = 0;
  std::vector<PinnableSlice> values(num_records);
  GetMergeOperandsOptions merge_operands_info;
  merge_operands_info.expected_max_number_of_operands = num_records;

  // k0 value in memtable
  Put("k0", "PutARock");
  db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(), "k0",
                        values.data(), &merge_operands_info,
                        &number_of_operands);
  ASSERT_EQ(values[0], "PutARock");

  // k0.1 value in SST
  Put("k0.1", "RockInSST");
  ASSERT_OK(Flush());
  db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(), "k0.1",
                        values.data(), &merge_operands_info,
                        &number_of_operands);
  ASSERT_EQ(values[0], "RockInSST");

  // All k1 values are in memtable.
  ASSERT_OK(Merge("k1", "a"));
  Put("k1", "x");
  ASSERT_OK(Merge("k1", "b"));
  ASSERT_OK(Merge("k1", "c"));
  ASSERT_OK(Merge("k1", "d"));
  db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(), "k1",
                        values.data(), &merge_operands_info,
                        &number_of_operands);
  ASSERT_EQ(values[0], "x");
  ASSERT_EQ(values[1], "b");
  ASSERT_EQ(values[2], "c");
  ASSERT_EQ(values[3], "d");

  // expected_max_number_of_operands is less than number of merge operands so
  // status should be Incomplete.
  merge_operands_info.expected_max_number_of_operands = num_records - 1;
  Status status = db_->GetMergeOperands(
      ReadOptions(), db_->DefaultColumnFamily(), "k1", values.data(),
      &merge_operands_info, &number_of_operands);
  ASSERT_EQ(status.IsIncomplete(), true);
  merge_operands_info.expected_max_number_of_operands = num_records;

  // All k1.1 values are in memtable.
  ASSERT_OK(Merge("k1.1", "r"));
  Delete("k1.1");
  ASSERT_OK(Merge("k1.1", "c"));
  ASSERT_OK(Merge("k1.1", "k"));
  ASSERT_OK(Merge("k1.1", "s"));
  db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(), "k1.1",
                        values.data(), &merge_operands_info,
                        &number_of_operands);
  ASSERT_EQ(values[0], "c");
  ASSERT_EQ(values[1], "k");
  ASSERT_EQ(values[2], "s");

  // All k2 values are flushed to L0 into a single file.
  ASSERT_OK(Merge("k2", "q"));
  ASSERT_OK(Merge("k2", "w"));
  ASSERT_OK(Merge("k2", "e"));
  ASSERT_OK(Merge("k2", "r"));
  ASSERT_OK(Flush());
  db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(), "k2",
                        values.data(), &merge_operands_info,
                        &number_of_operands);
  ASSERT_EQ(values[0], "q");
  ASSERT_EQ(values[1], "w");
  ASSERT_EQ(values[2], "e");
  ASSERT_EQ(values[3], "r");

  // All k2.1 values are flushed to L0 into a single file.
  ASSERT_OK(Merge("k2.1", "m"));
  Put("k2.1", "l");
  ASSERT_OK(Merge("k2.1", "n"));
  ASSERT_OK(Merge("k2.1", "o"));
  ASSERT_OK(Flush());
  db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(), "k2.1",
                        values.data(), &merge_operands_info,
                        &number_of_operands);
  ASSERT_EQ(values[0], "l,n,o");

  // All k2.2 values are flushed to L0 into a single file.
  ASSERT_OK(Merge("k2.2", "g"));
  Delete("k2.2");
  ASSERT_OK(Merge("k2.2", "o"));
  ASSERT_OK(Merge("k2.2", "t"));
  ASSERT_OK(Flush());
  db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(), "k2.2",
                        values.data(), &merge_operands_info,
                        &number_of_operands);
  ASSERT_EQ(values[0], "o,t");

  // Do some compaction that will make the following tests more predictable
  //  Slice start("PutARock");
  //  Slice end("t");
  db_->CompactRange(CompactRangeOptions(), nullptr, nullptr);

  // All k3 values are flushed and are in different files.
  ASSERT_OK(Merge("k3", "ab"));
  ASSERT_OK(Flush());
  ASSERT_OK(Merge("k3", "bc"));
  ASSERT_OK(Flush());
  ASSERT_OK(Merge("k3", "cd"));
  ASSERT_OK(Flush());
  ASSERT_OK(Merge("k3", "de"));
  db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(), "k3",
                        values.data(), &merge_operands_info,
                        &number_of_operands);
  ASSERT_EQ(values[0], "ab");
  ASSERT_EQ(values[1], "bc");
  ASSERT_EQ(values[2], "cd");
  ASSERT_EQ(values[3], "de");

  // All k3.1 values are flushed and are in different files.
  ASSERT_OK(Merge("k3.1", "ab"));
  ASSERT_OK(Flush());
  Put("k3.1", "bc");
  ASSERT_OK(Flush());
  ASSERT_OK(Merge("k3.1", "cd"));
  ASSERT_OK(Flush());
  ASSERT_OK(Merge("k3.1", "de"));
  db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(), "k3.1",
                        values.data(), &merge_operands_info,
                        &number_of_operands);
  ASSERT_EQ(values[0], "bc");
  ASSERT_EQ(values[1], "cd");
  ASSERT_EQ(values[2], "de");

  // All k3.2 values are flushed and are in different files.
  ASSERT_OK(Merge("k3.2", "ab"));
  ASSERT_OK(Flush());
  Delete("k3.2");
  ASSERT_OK(Flush());
  ASSERT_OK(Merge("k3.2", "cd"));
  ASSERT_OK(Flush());
  ASSERT_OK(Merge("k3.2", "de"));
  db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(), "k3.2",
                        values.data(), &merge_operands_info,
                        &number_of_operands);
  ASSERT_EQ(values[0], "cd");
  ASSERT_EQ(values[1], "de");

  // All K4 values are in different levels
  ASSERT_OK(Merge("k4", "ba"));
  ASSERT_OK(Flush());
  MoveFilesToLevel(4);
  ASSERT_OK(Merge("k4", "cb"));
  ASSERT_OK(Flush());
  MoveFilesToLevel(3);
  ASSERT_OK(Merge("k4", "dc"));
  ASSERT_OK(Flush());
  MoveFilesToLevel(1);
  ASSERT_OK(Merge("k4", "ed"));
  db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(), "k4",
                        values.data(), &merge_operands_info,
                        &number_of_operands);
  ASSERT_EQ(values[0], "ba");
  ASSERT_EQ(values[1], "cb");
  ASSERT_EQ(values[2], "dc");
  ASSERT_EQ(values[3], "ed");

  // First 3 k5 values are in SST and next 4 k5 values are in Immutable Memtable
  ASSERT_OK(Merge("k5", "who"));
  ASSERT_OK(Merge("k5", "am"));
  ASSERT_OK(Merge("k5", "i"));
  ASSERT_OK(Flush());
  Put("k5", "remember");
  ASSERT_OK(Merge("k5", "i"));
  ASSERT_OK(Merge("k5", "am"));
  ASSERT_OK(Merge("k5", "rocks"));
  dbfull()->TEST_SwitchMemtable();
  db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(), "k5",
                        values.data(), &merge_operands_info,
                        &number_of_operands);
  ASSERT_EQ(values[0], "remember");
  ASSERT_EQ(values[1], "i");
  ASSERT_EQ(values[2], "am");
}

}  // namespace ROCKSDB_NAMESPACE

int main(int argc, char** argv) {
  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
  ::testing::InitGoogleTest(&argc, argv);
  return RUN_ALL_TESTS();
}
Commit	Line	Data
f67539c2 TL	1	// Copyright (c) 2018-present, Facebook, Inc. All rights reserved.
	2	// This source code is licensed under both the GPLv2 (found in the
	3	// COPYING file in the root directory) and Apache 2.0 License
	4	// (found in the LICENSE.Apache file in the root directory).
	5
	6	#include "db/db_test_util.h"
	7	#include "port/stack_trace.h"
	8	#include "rocksdb/perf_context.h"
	9	#include "rocksdb/utilities/debug.h"
	10	#include "table/block_based/block_builder.h"
f67539c2 TL	11	#if !defined(ROCKSDB_LITE)
	12	#include "test_util/sync_point.h"
	13	#endif
	14	#include "rocksdb/merge_operator.h"
20effc67	15	#include "utilities/fault_injection_env.h"
f67539c2 TL	16	#include "utilities/merge_operators.h"
	17	#include "utilities/merge_operators/sortlist.h"
	18	#include "utilities/merge_operators/string_append/stringappend2.h"
	19
	20	namespace ROCKSDB_NAMESPACE {
	21
	22	class DBMergeOperandTest : public DBTestBase {
	23	public:
20effc67 TL	24	DBMergeOperandTest()
20effc67 TL	25	: DBTestBase("/db_merge_operand_test", /env_do_fsync=/true) {}
f67539c2 TL	26	};
	27
	28	TEST_F(DBMergeOperandTest, GetMergeOperandsBasic) {
	29	class LimitedStringAppendMergeOp : public StringAppendTESTOperator {
	30	public:
	31	LimitedStringAppendMergeOp(int limit, char delim)
	32	: StringAppendTESTOperator(delim), limit_(limit) {}
	33
	34	const char* Name() const override {
	35	return "DBMergeOperatorTest::LimitedStringAppendMergeOp";
	36	}
	37
	38	bool ShouldMerge(const std::vector<Slice>& operands) const override {
	39	if (operands.size() > 0 && limit_ > 0 && operands.size() >= limit_) {
	40	return true;
	41	}
	42	return false;
	43	}
	44
	45	private:
	46	size_t limit_ = 0;
	47	};
	48
	49	Options options;
	50	options.create_if_missing = true;
	51	// Use only the latest two merge operands.
	52	options.merge_operator = std::make_shared<LimitedStringAppendMergeOp>(2, ',');
	53	options.env = env_;
	54	Reopen(options);
	55	int num_records = 4;
	56	int number_of_operands = 0;
	57	std::vector<PinnableSlice> values(num_records);
	58	GetMergeOperandsOptions merge_operands_info;
	59	merge_operands_info.expected_max_number_of_operands = num_records;
	60
	61	// k0 value in memtable
	62	Put("k0", "PutARock");
	63	db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(), "k0",
	64	values.data(), &merge_operands_info,
	65	&number_of_operands);
	66	ASSERT_EQ(values[0], "PutARock");
	67
	68	// k0.1 value in SST
	69	Put("k0.1", "RockInSST");
	70	ASSERT_OK(Flush());
	71	db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(), "k0.1",
	72	values.data(), &merge_operands_info,
	73	&number_of_operands);
	74	ASSERT_EQ(values[0], "RockInSST");
	75
	76	// All k1 values are in memtable.
	77	ASSERT_OK(Merge("k1", "a"));
	78	Put("k1", "x");
	79	ASSERT_OK(Merge("k1", "b"));
	80	ASSERT_OK(Merge("k1", "c"));
	81	ASSERT_OK(Merge("k1", "d"));
	82	db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(), "k1",
	83	values.data(), &merge_operands_info,
	84	&number_of_operands);
	85	ASSERT_EQ(values[0], "x");
	86	ASSERT_EQ(values[1], "b");
	87	ASSERT_EQ(values[2], "c");
	88	ASSERT_EQ(values[3], "d");
	89
90	// expected_max_number_of_operands is less than number of merge operands so
91	// status should be Incomplete.
92	merge_operands_info.expected_max_number_of_operands = num_records - 1;
93	Status status = db_->GetMergeOperands(
94	ReadOptions(), db_->DefaultColumnFamily(), "k1", values.data(),
95	&merge_operands_info, &number_of_operands);
96	ASSERT_EQ(status.IsIncomplete(), true);
97	merge_operands_info.expected_max_number_of_operands = num_records;
98
99	// All k1.1 values are in memtable.
100	ASSERT_OK(Merge("k1.1", "r"));
101	Delete("k1.1");
102	ASSERT_OK(Merge("k1.1", "c"));
103	ASSERT_OK(Merge("k1.1", "k"));
104	ASSERT_OK(Merge("k1.1", "s"));
105	db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(), "k1.1",
106	values.data(), &merge_operands_info,
107	&number_of_operands);
108	ASSERT_EQ(values[0], "c");
109	ASSERT_EQ(values[1], "k");
110	ASSERT_EQ(values[2], "s");
111
112	// All k2 values are flushed to L0 into a single file.
113	ASSERT_OK(Merge("k2", "q"));
114	ASSERT_OK(Merge("k2", "w"));
115	ASSERT_OK(Merge("k2", "e"));
116	ASSERT_OK(Merge("k2", "r"));
117	ASSERT_OK(Flush());
118	db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(), "k2",
119	values.data(), &merge_operands_info,
120	&number_of_operands);
121	ASSERT_EQ(values[0], "q");
122	ASSERT_EQ(values[1], "w");
123	ASSERT_EQ(values[2], "e");
124	ASSERT_EQ(values[3], "r");
125
126	// All k2.1 values are flushed to L0 into a single file.
127	ASSERT_OK(Merge("k2.1", "m"));
128	Put("k2.1", "l");
129	ASSERT_OK(Merge("k2.1", "n"));
130	ASSERT_OK(Merge("k2.1", "o"));
131	ASSERT_OK(Flush());
132	db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(), "k2.1",
133	values.data(), &merge_operands_info,
134	&number_of_operands);
135	ASSERT_EQ(values[0], "l,n,o");
136
137	// All k2.2 values are flushed to L0 into a single file.
138	ASSERT_OK(Merge("k2.2", "g"));
139	Delete("k2.2");
140	ASSERT_OK(Merge("k2.2", "o"));
141	ASSERT_OK(Merge("k2.2", "t"));
142	ASSERT_OK(Flush());
143	db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(), "k2.2",
144	values.data(), &merge_operands_info,
145	&number_of_operands);
146	ASSERT_EQ(values[0], "o,t");
147
148	// Do some compaction that will make the following tests more predictable
149	// Slice start("PutARock");
150	// Slice end("t");
151	db_->CompactRange(CompactRangeOptions(), nullptr, nullptr);
152
153	// All k3 values are flushed and are in different files.
154	ASSERT_OK(Merge("k3", "ab"));
155	ASSERT_OK(Flush());
156	ASSERT_OK(Merge("k3", "bc"));
157	ASSERT_OK(Flush());
158	ASSERT_OK(Merge("k3", "cd"));
159	ASSERT_OK(Flush());
160	ASSERT_OK(Merge("k3", "de"));
161	db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(), "k3",
162	values.data(), &merge_operands_info,
163	&number_of_operands);
164	ASSERT_EQ(values[0], "ab");
165	ASSERT_EQ(values[1], "bc");
166	ASSERT_EQ(values[2], "cd");
167	ASSERT_EQ(values[3], "de");
168
169	// All k3.1 values are flushed and are in different files.
170	ASSERT_OK(Merge("k3.1", "ab"));
171	ASSERT_OK(Flush());
172	Put("k3.1", "bc");
173	ASSERT_OK(Flush());
174	ASSERT_OK(Merge("k3.1", "cd"));
175	ASSERT_OK(Flush());
176	ASSERT_OK(Merge("k3.1", "de"));
177	db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(), "k3.1",
178	values.data(), &merge_operands_info,
179	&number_of_operands);
180	ASSERT_EQ(values[0], "bc");
181	ASSERT_EQ(values[1], "cd");
182	ASSERT_EQ(values[2], "de");
183
184	// All k3.2 values are flushed and are in different files.
185	ASSERT_OK(Merge("k3.2", "ab"));
186	ASSERT_OK(Flush());
187	Delete("k3.2");
188	ASSERT_OK(Flush());
189	ASSERT_OK(Merge("k3.2", "cd"));
190	ASSERT_OK(Flush());
191	ASSERT_OK(Merge("k3.2", "de"));
192	db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(), "k3.2",
193	values.data(), &merge_operands_info,
194	&number_of_operands);
195	ASSERT_EQ(values[0], "cd");
196	ASSERT_EQ(values[1], "de");
197
198	// All K4 values are in different levels
199	ASSERT_OK(Merge("k4", "ba"));
200	ASSERT_OK(Flush());
201	MoveFilesToLevel(4);
202	ASSERT_OK(Merge("k4", "cb"));
203	ASSERT_OK(Flush());
204	MoveFilesToLevel(3);
205	ASSERT_OK(Merge("k4", "dc"));
206	ASSERT_OK(Flush());
207	MoveFilesToLevel(1);
208	ASSERT_OK(Merge("k4", "ed"));
209	db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(), "k4",
210	values.data(), &merge_operands_info,
211	&number_of_operands);
212	ASSERT_EQ(values[0], "ba");
213	ASSERT_EQ(values[1], "cb");
214	ASSERT_EQ(values[2], "dc");
215	ASSERT_EQ(values[3], "ed");
216
217	// First 3 k5 values are in SST and next 4 k5 values are in Immutable Memtable
218	ASSERT_OK(Merge("k5", "who"));
219	ASSERT_OK(Merge("k5", "am"));
220	ASSERT_OK(Merge("k5", "i"));
221	ASSERT_OK(Flush());
222	Put("k5", "remember");
223	ASSERT_OK(Merge("k5", "i"));
224	ASSERT_OK(Merge("k5", "am"));
225	ASSERT_OK(Merge("k5", "rocks"));
226	dbfull()->TEST_SwitchMemtable();
227	db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(), "k5",
228	values.data(), &merge_operands_info,
229	&number_of_operands);
230	ASSERT_EQ(values[0], "remember");
231	ASSERT_EQ(values[1], "i");
232	ASSERT_EQ(values[2], "am");
233	}
234
235	} // namespace ROCKSDB_NAMESPACE
236
237	int main(int argc, char** argv) {
238	ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
239	::testing::InitGoogleTest(&argc, argv);
240	return RUN_ALL_TESTS();
241	}