]>
Commit | Line | Data |
---|---|---|
7c673cae | 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. |
11fdf7f2 TL |
2 | // This source code is licensed under both the GPLv2 (found in the |
3 | // COPYING file in the root directory) and Apache 2.0 License | |
4 | // (found in the LICENSE.Apache file in the root directory). | |
7c673cae FG |
5 | |
6 | #ifndef ROCKSDB_LITE | |
7 | ||
1e59de90 TL |
8 | #include "db/wal_manager.h" |
9 | ||
7c673cae FG |
10 | #include <map> |
11 | #include <string> | |
12 | ||
7c673cae | 13 | #include "db/column_family.h" |
f67539c2 | 14 | #include "db/db_impl/db_impl.h" |
7c673cae FG |
15 | #include "db/log_writer.h" |
16 | #include "db/version_set.h" | |
7c673cae | 17 | #include "env/mock_env.h" |
f67539c2 | 18 | #include "file/writable_file_writer.h" |
1e59de90 TL |
19 | #include "rocksdb/cache.h" |
20 | #include "rocksdb/file_system.h" | |
21 | #include "rocksdb/write_batch.h" | |
22 | #include "rocksdb/write_buffer_manager.h" | |
7c673cae | 23 | #include "table/mock_table.h" |
f67539c2 TL |
24 | #include "test_util/testharness.h" |
25 | #include "test_util/testutil.h" | |
7c673cae | 26 | #include "util/string_util.h" |
7c673cae | 27 | |
f67539c2 | 28 | namespace ROCKSDB_NAMESPACE { |
7c673cae FG |
29 | |
30 | // TODO(icanadi) mock out VersionSet | |
31 | // TODO(icanadi) move other WalManager-specific tests from db_test here | |
32 | class WalManagerTest : public testing::Test { | |
33 | public: | |
34 | WalManagerTest() | |
1e59de90 | 35 | : dbname_(test::PerThreadDBPath("wal_manager_test")), |
7c673cae FG |
36 | db_options_(), |
37 | table_cache_(NewLRUCache(50000, 16)), | |
38 | write_buffer_manager_(db_options_.db_write_buffer_size), | |
39 | current_log_number_(0) { | |
1e59de90 TL |
40 | env_.reset(MockEnv::Create(Env::Default())); |
41 | EXPECT_OK(DestroyDB(dbname_, Options())); | |
7c673cae FG |
42 | } |
43 | ||
44 | void Init() { | |
45 | ASSERT_OK(env_->CreateDirIfMissing(dbname_)); | |
46 | ASSERT_OK(env_->CreateDirIfMissing(ArchivalDirectory(dbname_))); | |
47 | db_options_.db_paths.emplace_back(dbname_, | |
48 | std::numeric_limits<uint64_t>::max()); | |
49 | db_options_.wal_dir = dbname_; | |
50 | db_options_.env = env_.get(); | |
1e59de90 TL |
51 | db_options_.fs = env_->GetFileSystem(); |
52 | db_options_.clock = env_->GetSystemClock().get(); | |
7c673cae | 53 | |
20effc67 TL |
54 | versions_.reset( |
55 | new VersionSet(dbname_, &db_options_, env_options_, table_cache_.get(), | |
56 | &write_buffer_manager_, &write_controller_, | |
1e59de90 TL |
57 | /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr, |
58 | /*db_id*/ "", /*db_session_id*/ "")); | |
7c673cae | 59 | |
20effc67 TL |
60 | wal_manager_.reset( |
61 | new WalManager(db_options_, env_options_, nullptr /*IOTracer*/)); | |
7c673cae FG |
62 | } |
63 | ||
64 | void Reopen() { | |
20effc67 TL |
65 | wal_manager_.reset( |
66 | new WalManager(db_options_, env_options_, nullptr /*IOTracer*/)); | |
7c673cae FG |
67 | } |
68 | ||
69 | // NOT thread safe | |
70 | void Put(const std::string& key, const std::string& value) { | |
71 | assert(current_log_writer_.get() != nullptr); | |
1e59de90 | 72 | uint64_t seq = versions_->LastSequence() + 1; |
7c673cae | 73 | WriteBatch batch; |
1e59de90 | 74 | ASSERT_OK(batch.Put(key, value)); |
7c673cae | 75 | WriteBatchInternal::SetSequence(&batch, seq); |
1e59de90 TL |
76 | ASSERT_OK( |
77 | current_log_writer_->AddRecord(WriteBatchInternal::Contents(&batch))); | |
11fdf7f2 TL |
78 | versions_->SetLastAllocatedSequence(seq); |
79 | versions_->SetLastPublishedSequence(seq); | |
7c673cae FG |
80 | versions_->SetLastSequence(seq); |
81 | } | |
82 | ||
83 | // NOT thread safe | |
11fdf7f2 | 84 | void RollTheLog(bool /*archived*/) { |
7c673cae FG |
85 | current_log_number_++; |
86 | std::string fname = ArchivedLogFileName(dbname_, current_log_number_); | |
1e59de90 TL |
87 | const auto& fs = env_->GetFileSystem(); |
88 | std::unique_ptr<WritableFileWriter> file_writer; | |
89 | ASSERT_OK(WritableFileWriter::Create(fs, fname, env_options_, &file_writer, | |
90 | nullptr)); | |
91 | current_log_writer_.reset( | |
92 | new log::Writer(std::move(file_writer), 0, false)); | |
7c673cae FG |
93 | } |
94 | ||
95 | void CreateArchiveLogs(int num_logs, int entries_per_log) { | |
96 | for (int i = 1; i <= num_logs; ++i) { | |
97 | RollTheLog(true); | |
98 | for (int k = 0; k < entries_per_log; ++k) { | |
1e59de90 | 99 | Put(std::to_string(k), std::string(1024, 'a')); |
7c673cae FG |
100 | } |
101 | } | |
102 | } | |
103 | ||
104 | std::unique_ptr<TransactionLogIterator> OpenTransactionLogIter( | |
105 | const SequenceNumber seq) { | |
494da23a | 106 | std::unique_ptr<TransactionLogIterator> iter; |
7c673cae FG |
107 | Status status = wal_manager_->GetUpdatesSince( |
108 | seq, &iter, TransactionLogIterator::ReadOptions(), versions_.get()); | |
109 | EXPECT_OK(status); | |
110 | return iter; | |
111 | } | |
112 | ||
113 | std::unique_ptr<MockEnv> env_; | |
114 | std::string dbname_; | |
115 | ImmutableDBOptions db_options_; | |
116 | WriteController write_controller_; | |
117 | EnvOptions env_options_; | |
118 | std::shared_ptr<Cache> table_cache_; | |
119 | WriteBufferManager write_buffer_manager_; | |
120 | std::unique_ptr<VersionSet> versions_; | |
121 | std::unique_ptr<WalManager> wal_manager_; | |
122 | ||
123 | std::unique_ptr<log::Writer> current_log_writer_; | |
124 | uint64_t current_log_number_; | |
125 | }; | |
126 | ||
127 | TEST_F(WalManagerTest, ReadFirstRecordCache) { | |
128 | Init(); | |
129 | std::string path = dbname_ + "/000001.log"; | |
1e59de90 TL |
130 | std::unique_ptr<FSWritableFile> file; |
131 | ASSERT_OK(env_->GetFileSystem()->NewWritableFile(path, FileOptions(), &file, | |
132 | nullptr)); | |
7c673cae FG |
133 | |
134 | SequenceNumber s; | |
135 | ASSERT_OK(wal_manager_->TEST_ReadFirstLine(path, 1 /* number */, &s)); | |
136 | ASSERT_EQ(s, 0U); | |
137 | ||
138 | ASSERT_OK( | |
139 | wal_manager_->TEST_ReadFirstRecord(kAliveLogFile, 1 /* number */, &s)); | |
140 | ASSERT_EQ(s, 0U); | |
141 | ||
1e59de90 TL |
142 | std::unique_ptr<WritableFileWriter> file_writer( |
143 | new WritableFileWriter(std::move(file), path, FileOptions())); | |
7c673cae FG |
144 | log::Writer writer(std::move(file_writer), 1, |
145 | db_options_.recycle_log_file_num > 0); | |
146 | WriteBatch batch; | |
1e59de90 | 147 | ASSERT_OK(batch.Put("foo", "bar")); |
7c673cae | 148 | WriteBatchInternal::SetSequence(&batch, 10); |
1e59de90 | 149 | ASSERT_OK(writer.AddRecord(WriteBatchInternal::Contents(&batch))); |
7c673cae FG |
150 | |
151 | // TODO(icanadi) move SpecialEnv outside of db_test, so we can reuse it here. | |
152 | // Waiting for lei to finish with db_test | |
153 | // env_->count_sequential_reads_ = true; | |
154 | // sequential_read_counter_ sanity test | |
155 | // ASSERT_EQ(env_->sequential_read_counter_.Read(), 0); | |
156 | ||
157 | ASSERT_OK(wal_manager_->TEST_ReadFirstRecord(kAliveLogFile, 1, &s)); | |
158 | ASSERT_EQ(s, 10U); | |
159 | // did a read | |
160 | // TODO(icanadi) move SpecialEnv outside of db_test, so we can reuse it here | |
161 | // ASSERT_EQ(env_->sequential_read_counter_.Read(), 1); | |
162 | ||
163 | ASSERT_OK(wal_manager_->TEST_ReadFirstRecord(kAliveLogFile, 1, &s)); | |
164 | ASSERT_EQ(s, 10U); | |
165 | // no new reads since the value is cached | |
166 | // TODO(icanadi) move SpecialEnv outside of db_test, so we can reuse it here | |
167 | // ASSERT_EQ(env_->sequential_read_counter_.Read(), 1); | |
168 | } | |
169 | ||
170 | namespace { | |
171 | uint64_t GetLogDirSize(std::string dir_path, Env* env) { | |
172 | uint64_t dir_size = 0; | |
173 | std::vector<std::string> files; | |
1e59de90 | 174 | EXPECT_OK(env->GetChildren(dir_path, &files)); |
7c673cae FG |
175 | for (auto& f : files) { |
176 | uint64_t number; | |
177 | FileType type; | |
20effc67 | 178 | if (ParseFileName(f, &number, &type) && type == kWalFile) { |
7c673cae FG |
179 | std::string const file_path = dir_path + "/" + f; |
180 | uint64_t file_size; | |
1e59de90 | 181 | EXPECT_OK(env->GetFileSize(file_path, &file_size)); |
7c673cae FG |
182 | dir_size += file_size; |
183 | } | |
184 | } | |
185 | return dir_size; | |
186 | } | |
187 | std::vector<std::uint64_t> ListSpecificFiles( | |
188 | Env* env, const std::string& path, const FileType expected_file_type) { | |
189 | std::vector<std::string> files; | |
190 | std::vector<uint64_t> file_numbers; | |
7c673cae FG |
191 | uint64_t number; |
192 | FileType type; | |
1e59de90 | 193 | EXPECT_OK(env->GetChildren(path, &files)); |
7c673cae FG |
194 | for (size_t i = 0; i < files.size(); ++i) { |
195 | if (ParseFileName(files[i], &number, &type)) { | |
196 | if (type == expected_file_type) { | |
197 | file_numbers.push_back(number); | |
198 | } | |
199 | } | |
200 | } | |
201 | return file_numbers; | |
202 | } | |
203 | ||
204 | int CountRecords(TransactionLogIterator* iter) { | |
205 | int count = 0; | |
206 | SequenceNumber lastSequence = 0; | |
207 | BatchResult res; | |
208 | while (iter->Valid()) { | |
209 | res = iter->GetBatch(); | |
210 | EXPECT_TRUE(res.sequence > lastSequence); | |
211 | ++count; | |
212 | lastSequence = res.sequence; | |
213 | EXPECT_OK(iter->status()); | |
214 | iter->Next(); | |
215 | } | |
1e59de90 | 216 | EXPECT_OK(iter->status()); |
7c673cae FG |
217 | return count; |
218 | } | |
1e59de90 | 219 | } // anonymous namespace |
7c673cae FG |
220 | |
221 | TEST_F(WalManagerTest, WALArchivalSizeLimit) { | |
1e59de90 TL |
222 | db_options_.WAL_ttl_seconds = 0; |
223 | db_options_.WAL_size_limit_MB = 1000; | |
7c673cae FG |
224 | Init(); |
225 | ||
226 | // TEST : Create WalManager with huge size limit and no ttl. | |
227 | // Create some archived files and call PurgeObsoleteWALFiles(). | |
228 | // Count the archived log files that survived. | |
229 | // Assert that all of them did. | |
230 | // Change size limit. Re-open WalManager. | |
1e59de90 | 231 | // Assert that archive is not greater than WAL_size_limit_MB after |
7c673cae FG |
232 | // PurgeObsoleteWALFiles() |
233 | // Set ttl and time_to_check_ to small values. Re-open db. | |
234 | // Assert that there are no archived logs left. | |
235 | ||
236 | std::string archive_dir = ArchivalDirectory(dbname_); | |
237 | CreateArchiveLogs(20, 5000); | |
238 | ||
239 | std::vector<std::uint64_t> log_files = | |
20effc67 | 240 | ListSpecificFiles(env_.get(), archive_dir, kWalFile); |
7c673cae FG |
241 | ASSERT_EQ(log_files.size(), 20U); |
242 | ||
1e59de90 | 243 | db_options_.WAL_size_limit_MB = 8; |
7c673cae FG |
244 | Reopen(); |
245 | wal_manager_->PurgeObsoleteWALFiles(); | |
246 | ||
247 | uint64_t archive_size = GetLogDirSize(archive_dir, env_.get()); | |
1e59de90 | 248 | ASSERT_TRUE(archive_size <= db_options_.WAL_size_limit_MB * 1024 * 1024); |
7c673cae | 249 | |
1e59de90 TL |
250 | db_options_.WAL_ttl_seconds = 1; |
251 | env_->SleepForMicroseconds(2 * 1000 * 1000); | |
7c673cae FG |
252 | Reopen(); |
253 | wal_manager_->PurgeObsoleteWALFiles(); | |
254 | ||
20effc67 | 255 | log_files = ListSpecificFiles(env_.get(), archive_dir, kWalFile); |
7c673cae FG |
256 | ASSERT_TRUE(log_files.empty()); |
257 | } | |
258 | ||
259 | TEST_F(WalManagerTest, WALArchivalTtl) { | |
1e59de90 | 260 | db_options_.WAL_ttl_seconds = 1000; |
7c673cae FG |
261 | Init(); |
262 | ||
263 | // TEST : Create WalManager with a ttl and no size limit. | |
264 | // Create some archived log files and call PurgeObsoleteWALFiles(). | |
265 | // Assert that files are not deleted | |
266 | // Reopen db with small ttl. | |
267 | // Assert that all archived logs was removed. | |
268 | ||
269 | std::string archive_dir = ArchivalDirectory(dbname_); | |
270 | CreateArchiveLogs(20, 5000); | |
271 | ||
272 | std::vector<uint64_t> log_files = | |
20effc67 | 273 | ListSpecificFiles(env_.get(), archive_dir, kWalFile); |
7c673cae FG |
274 | ASSERT_GT(log_files.size(), 0U); |
275 | ||
1e59de90 TL |
276 | db_options_.WAL_ttl_seconds = 1; |
277 | env_->SleepForMicroseconds(3 * 1000 * 1000); | |
7c673cae FG |
278 | Reopen(); |
279 | wal_manager_->PurgeObsoleteWALFiles(); | |
280 | ||
20effc67 | 281 | log_files = ListSpecificFiles(env_.get(), archive_dir, kWalFile); |
7c673cae FG |
282 | ASSERT_TRUE(log_files.empty()); |
283 | } | |
284 | ||
285 | TEST_F(WalManagerTest, TransactionLogIteratorMoveOverZeroFiles) { | |
286 | Init(); | |
287 | RollTheLog(false); | |
288 | Put("key1", std::string(1024, 'a')); | |
289 | // Create a zero record WAL file. | |
290 | RollTheLog(false); | |
291 | RollTheLog(false); | |
292 | ||
293 | Put("key2", std::string(1024, 'a')); | |
294 | ||
295 | auto iter = OpenTransactionLogIter(0); | |
296 | ASSERT_EQ(2, CountRecords(iter.get())); | |
297 | } | |
298 | ||
299 | TEST_F(WalManagerTest, TransactionLogIteratorJustEmptyFile) { | |
300 | Init(); | |
301 | RollTheLog(false); | |
302 | auto iter = OpenTransactionLogIter(0); | |
303 | // Check that an empty iterator is returned | |
304 | ASSERT_TRUE(!iter->Valid()); | |
305 | } | |
306 | ||
f67539c2 TL |
307 | TEST_F(WalManagerTest, TransactionLogIteratorNewFileWhileScanning) { |
308 | Init(); | |
309 | CreateArchiveLogs(2, 100); | |
310 | auto iter = OpenTransactionLogIter(0); | |
311 | CreateArchiveLogs(1, 100); | |
312 | int i = 0; | |
313 | for (; iter->Valid(); iter->Next()) { | |
314 | i++; | |
315 | } | |
316 | ASSERT_EQ(i, 200); | |
317 | // A new log file was added after the iterator was created. | |
318 | // TryAgain indicates a new iterator is needed to fetch the new data | |
319 | ASSERT_TRUE(iter->status().IsTryAgain()); | |
320 | ||
321 | iter = OpenTransactionLogIter(0); | |
322 | i = 0; | |
323 | for (; iter->Valid(); iter->Next()) { | |
324 | i++; | |
325 | } | |
326 | ASSERT_EQ(i, 300); | |
327 | ASSERT_TRUE(iter->status().ok()); | |
328 | } | |
329 | ||
330 | } // namespace ROCKSDB_NAMESPACE | |
7c673cae FG |
331 | |
332 | int main(int argc, char** argv) { | |
1e59de90 | 333 | ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); |
7c673cae FG |
334 | ::testing::InitGoogleTest(&argc, argv); |
335 | return RUN_ALL_TESTS(); | |
336 | } | |
337 | ||
338 | #else | |
339 | #include <stdio.h> | |
340 | ||
11fdf7f2 | 341 | int main(int /*argc*/, char** /*argv*/) { |
7c673cae FG |
342 | fprintf(stderr, "SKIPPED as WalManager is not supported in ROCKSDB_LITE\n"); |
343 | return 0; | |
344 | } | |
345 | ||
346 | #endif // !ROCKSDB_LITE |