]> git.proxmox.com Git - ceph.git/blame - ceph/src/rocksdb/db/db_log_iter_test.cc
bump version to 18.2.4-pve3
[ceph.git] / ceph / src / rocksdb / db / db_log_iter_test.cc
CommitLineData
7c673cae 1// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
11fdf7f2
TL
2// This source code is licensed under both the GPLv2 (found in the
3// COPYING file in the root directory) and Apache 2.0 License
4// (found in the LICENSE.Apache file in the root directory).
7c673cae
FG
5//
6// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
7// Use of this source code is governed by a BSD-style license that can be
8// found in the LICENSE file. See the AUTHORS file for names of contributors.
9
10// Introduction of SyncPoint effectively disabled building and running this test
11// in Release build.
12// which is a pity, it is a good test
13#if !defined(ROCKSDB_LITE)
14
15#include "db/db_test_util.h"
1e59de90 16#include "env/mock_env.h"
7c673cae
FG
17#include "port/stack_trace.h"
18
f67539c2 19namespace ROCKSDB_NAMESPACE {
7c673cae
FG
20
21class DBTestXactLogIterator : public DBTestBase {
22 public:
20effc67 23 DBTestXactLogIterator()
1e59de90 24 : DBTestBase("db_log_iter_test", /*env_do_fsync=*/true) {}
7c673cae
FG
25
26 std::unique_ptr<TransactionLogIterator> OpenTransactionLogIter(
27 const SequenceNumber seq) {
494da23a 28 std::unique_ptr<TransactionLogIterator> iter;
7c673cae
FG
29 Status status = dbfull()->GetUpdatesSince(seq, &iter);
30 EXPECT_OK(status);
31 EXPECT_TRUE(iter->Valid());
32 return iter;
33 }
34};
35
36namespace {
1e59de90
TL
37SequenceNumber ReadRecords(std::unique_ptr<TransactionLogIterator>& iter,
38 int& count, bool expect_ok = true) {
7c673cae
FG
39 count = 0;
40 SequenceNumber lastSequence = 0;
41 BatchResult res;
42 while (iter->Valid()) {
43 res = iter->GetBatch();
44 EXPECT_TRUE(res.sequence > lastSequence);
45 ++count;
46 lastSequence = res.sequence;
47 EXPECT_OK(iter->status());
48 iter->Next();
49 }
1e59de90
TL
50 if (expect_ok) {
51 EXPECT_OK(iter->status());
52 } else {
53 EXPECT_NOK(iter->status());
54 }
7c673cae
FG
55 return res.sequence;
56}
57
1e59de90
TL
58void ExpectRecords(const int expected_no_records,
59 std::unique_ptr<TransactionLogIterator>& iter) {
7c673cae
FG
60 int num_records;
61 ReadRecords(iter, num_records);
62 ASSERT_EQ(num_records, expected_no_records);
63}
1e59de90 64} // anonymous namespace
7c673cae
FG
65
66TEST_F(DBTestXactLogIterator, TransactionLogIterator) {
67 do {
68 Options options = OptionsForLogIterTest();
69 DestroyAndReopen(options);
70 CreateAndReopenWithCF({"pikachu"}, options);
1e59de90
TL
71 ASSERT_OK(Put(0, "key1", DummyString(1024)));
72 ASSERT_OK(Put(1, "key2", DummyString(1024)));
73 ASSERT_OK(Put(1, "key2", DummyString(1024)));
7c673cae
FG
74 ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 3U);
75 {
76 auto iter = OpenTransactionLogIter(0);
77 ExpectRecords(3, iter);
78 }
79 ReopenWithColumnFamilies({"default", "pikachu"}, options);
80 env_->SleepForMicroseconds(2 * 1000 * 1000);
81 {
1e59de90
TL
82 ASSERT_OK(Put(0, "key4", DummyString(1024)));
83 ASSERT_OK(Put(1, "key5", DummyString(1024)));
84 ASSERT_OK(Put(0, "key6", DummyString(1024)));
7c673cae
FG
85 }
86 {
87 auto iter = OpenTransactionLogIter(0);
88 ExpectRecords(6, iter);
89 }
90 } while (ChangeCompactOptions());
91}
92
93#ifndef NDEBUG // sync point is not included with DNDEBUG build
94TEST_F(DBTestXactLogIterator, TransactionLogIteratorRace) {
95 static const int LOG_ITERATOR_RACE_TEST_COUNT = 2;
96 static const char* sync_points[LOG_ITERATOR_RACE_TEST_COUNT][4] = {
1e59de90 97 {"WalManager::GetSortedWalFiles:1", "WalManager::PurgeObsoleteFiles:1",
7c673cae 98 "WalManager::PurgeObsoleteFiles:2", "WalManager::GetSortedWalFiles:2"},
1e59de90 99 {"WalManager::GetSortedWalsOfType:1", "WalManager::PurgeObsoleteFiles:1",
7c673cae
FG
100 "WalManager::PurgeObsoleteFiles:2",
101 "WalManager::GetSortedWalsOfType:2"}};
102 for (int test = 0; test < LOG_ITERATOR_RACE_TEST_COUNT; ++test) {
103 // Setup sync point dependency to reproduce the race condition of
104 // a log file moved to archived dir, in the middle of GetSortedWalFiles
f67539c2
TL
105 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({
106 {sync_points[test][0], sync_points[test][1]},
107 {sync_points[test][2], sync_points[test][3]},
108 });
7c673cae
FG
109
110 do {
f67539c2
TL
111 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearTrace();
112 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
7c673cae
FG
113 Options options = OptionsForLogIterTest();
114 DestroyAndReopen(options);
1e59de90
TL
115 ASSERT_OK(Put("key1", DummyString(1024)));
116 ASSERT_OK(dbfull()->Flush(FlushOptions()));
117 ASSERT_OK(Put("key2", DummyString(1024)));
118 ASSERT_OK(dbfull()->Flush(FlushOptions()));
119 ASSERT_OK(Put("key3", DummyString(1024)));
120 ASSERT_OK(dbfull()->Flush(FlushOptions()));
121 ASSERT_OK(Put("key4", DummyString(1024)));
7c673cae 122 ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 4U);
1e59de90 123 ASSERT_OK(dbfull()->FlushWAL(false));
7c673cae
FG
124
125 {
126 auto iter = OpenTransactionLogIter(0);
127 ExpectRecords(4, iter);
128 }
129
f67539c2 130 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
7c673cae
FG
131 // trigger async flush, and log move. Well, log move will
132 // wait until the GetSortedWalFiles:1 to reproduce the race
133 // condition
134 FlushOptions flush_options;
135 flush_options.wait = false;
1e59de90 136 ASSERT_OK(dbfull()->Flush(flush_options));
7c673cae
FG
137
138 // "key5" would be written in a new memtable and log
1e59de90
TL
139 ASSERT_OK(Put("key5", DummyString(1024)));
140 ASSERT_OK(dbfull()->FlushWAL(false));
7c673cae
FG
141 {
142 // this iter would miss "key4" if not fixed
143 auto iter = OpenTransactionLogIter(0);
144 ExpectRecords(5, iter);
145 }
146 } while (ChangeCompactOptions());
147 }
148}
149#endif
150
151TEST_F(DBTestXactLogIterator, TransactionLogIteratorStallAtLastRecord) {
152 do {
153 Options options = OptionsForLogIterTest();
154 DestroyAndReopen(options);
1e59de90 155 ASSERT_OK(Put("key1", DummyString(1024)));
7c673cae
FG
156 auto iter = OpenTransactionLogIter(0);
157 ASSERT_OK(iter->status());
158 ASSERT_TRUE(iter->Valid());
159 iter->Next();
160 ASSERT_TRUE(!iter->Valid());
161 ASSERT_OK(iter->status());
1e59de90 162 ASSERT_OK(Put("key2", DummyString(1024)));
7c673cae
FG
163 iter->Next();
164 ASSERT_OK(iter->status());
165 ASSERT_TRUE(iter->Valid());
166 } while (ChangeCompactOptions());
167}
168
169TEST_F(DBTestXactLogIterator, TransactionLogIteratorCheckAfterRestart) {
170 do {
171 Options options = OptionsForLogIterTest();
172 DestroyAndReopen(options);
1e59de90
TL
173 ASSERT_OK(Put("key1", DummyString(1024)));
174 ASSERT_OK(Put("key2", DummyString(1023)));
175 ASSERT_OK(dbfull()->Flush(FlushOptions()));
7c673cae
FG
176 Reopen(options);
177 auto iter = OpenTransactionLogIter(0);
178 ExpectRecords(2, iter);
179 } while (ChangeCompactOptions());
180}
181
182TEST_F(DBTestXactLogIterator, TransactionLogIteratorCorruptedLog) {
183 do {
184 Options options = OptionsForLogIterTest();
185 DestroyAndReopen(options);
1e59de90 186
7c673cae 187 for (int i = 0; i < 1024; i++) {
1e59de90 188 ASSERT_OK(Put("key" + std::to_string(i), DummyString(10)));
7c673cae 189 }
1e59de90
TL
190
191 ASSERT_OK(Flush());
192 ASSERT_OK(db_->FlushWAL(false));
193
7c673cae 194 // Corrupt this log to create a gap
1e59de90
TL
195 ASSERT_OK(db_->DisableFileDeletions());
196
197 VectorLogPtr wal_files;
198 ASSERT_OK(db_->GetSortedWalFiles(wal_files));
199 ASSERT_FALSE(wal_files.empty());
200
7c673cae 201 const auto logfile_path = dbname_ + "/" + wal_files.front()->PathName();
1e59de90
TL
202 ASSERT_OK(test::TruncateFile(env_, logfile_path,
203 wal_files.front()->SizeFileBytes() / 2));
204
205 ASSERT_OK(db_->EnableFileDeletions());
7c673cae
FG
206
207 // Insert a new entry to a new log file
1e59de90
TL
208 ASSERT_OK(Put("key1025", DummyString(10)));
209 ASSERT_OK(db_->FlushWAL(false));
210
7c673cae
FG
211 // Try to read from the beginning. Should stop before the gap and read less
212 // than 1025 entries
213 auto iter = OpenTransactionLogIter(0);
1e59de90
TL
214 int count = 0;
215 SequenceNumber last_sequence_read = ReadRecords(iter, count, false);
7c673cae 216 ASSERT_LT(last_sequence_read, 1025U);
1e59de90 217
7c673cae
FG
218 // Try to read past the gap, should be able to seek to key1025
219 auto iter2 = OpenTransactionLogIter(last_sequence_read + 1);
220 ExpectRecords(1, iter2);
221 } while (ChangeCompactOptions());
222}
223
224TEST_F(DBTestXactLogIterator, TransactionLogIteratorBatchOperations) {
225 do {
226 Options options = OptionsForLogIterTest();
227 DestroyAndReopen(options);
228 CreateAndReopenWithCF({"pikachu"}, options);
229 WriteBatch batch;
1e59de90
TL
230 ASSERT_OK(batch.Put(handles_[1], "key1", DummyString(1024)));
231 ASSERT_OK(batch.Put(handles_[0], "key2", DummyString(1024)));
232 ASSERT_OK(batch.Put(handles_[1], "key3", DummyString(1024)));
233 ASSERT_OK(batch.Delete(handles_[0], "key2"));
234 ASSERT_OK(dbfull()->Write(WriteOptions(), &batch));
235 ASSERT_OK(Flush(1));
236 ASSERT_OK(Flush(0));
7c673cae 237 ReopenWithColumnFamilies({"default", "pikachu"}, options);
1e59de90 238 ASSERT_OK(Put(1, "key4", DummyString(1024)));
7c673cae
FG
239 auto iter = OpenTransactionLogIter(3);
240 ExpectRecords(2, iter);
241 } while (ChangeCompactOptions());
242}
243
244TEST_F(DBTestXactLogIterator, TransactionLogIteratorBlobs) {
245 Options options = OptionsForLogIterTest();
246 DestroyAndReopen(options);
247 CreateAndReopenWithCF({"pikachu"}, options);
248 {
249 WriteBatch batch;
1e59de90
TL
250 ASSERT_OK(batch.Put(handles_[1], "key1", DummyString(1024)));
251 ASSERT_OK(batch.Put(handles_[0], "key2", DummyString(1024)));
252 ASSERT_OK(batch.PutLogData(Slice("blob1")));
253 ASSERT_OK(batch.Put(handles_[1], "key3", DummyString(1024)));
254 ASSERT_OK(batch.PutLogData(Slice("blob2")));
255 ASSERT_OK(batch.Delete(handles_[0], "key2"));
256 ASSERT_OK(dbfull()->Write(WriteOptions(), &batch));
7c673cae
FG
257 ReopenWithColumnFamilies({"default", "pikachu"}, options);
258 }
259
260 auto res = OpenTransactionLogIter(0)->GetBatch();
261 struct Handler : public WriteBatch::Handler {
262 std::string seen;
494da23a 263 Status PutCF(uint32_t cf, const Slice& key, const Slice& value) override {
1e59de90
TL
264 seen += "Put(" + std::to_string(cf) + ", " + key.ToString() + ", " +
265 std::to_string(value.size()) + ")";
7c673cae
FG
266 return Status::OK();
267 }
494da23a 268 Status MergeCF(uint32_t cf, const Slice& key, const Slice& value) override {
1e59de90
TL
269 seen += "Merge(" + std::to_string(cf) + ", " + key.ToString() + ", " +
270 std::to_string(value.size()) + ")";
7c673cae
FG
271 return Status::OK();
272 }
494da23a 273 void LogData(const Slice& blob) override {
7c673cae
FG
274 seen += "LogData(" + blob.ToString() + ")";
275 }
494da23a 276 Status DeleteCF(uint32_t cf, const Slice& key) override {
1e59de90 277 seen += "Delete(" + std::to_string(cf) + ", " + key.ToString() + ")";
7c673cae
FG
278 return Status::OK();
279 }
280 } handler;
1e59de90 281 ASSERT_OK(res.writeBatchPtr->Iterate(&handler));
7c673cae
FG
282 ASSERT_EQ(
283 "Put(1, key1, 1024)"
284 "Put(0, key2, 1024)"
285 "LogData(blob1)"
286 "Put(1, key3, 1024)"
287 "LogData(blob2)"
288 "Delete(0, key2)",
289 handler.seen);
290}
f67539c2 291} // namespace ROCKSDB_NAMESPACE
7c673cae
FG
292
293#endif // !defined(ROCKSDB_LITE)
294
295int main(int argc, char** argv) {
296#if !defined(ROCKSDB_LITE)
f67539c2 297 ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
7c673cae
FG
298 ::testing::InitGoogleTest(&argc, argv);
299 return RUN_ALL_TESTS();
300#else
1e59de90
TL
301 (void)argc;
302 (void)argv;
7c673cae
FG
303 return 0;
304#endif
305}