]> git.proxmox.com Git - ceph.git/blame - ceph/src/rocksdb/util/file_reader_writer_test.cc
import 14.2.4 nautilus point release
[ceph.git] / ceph / src / rocksdb / util / file_reader_writer_test.cc
CommitLineData
7c673cae 1// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
11fdf7f2
TL
2// This source code is licensed under both the GPLv2 (found in the
3// COPYING file in the root directory) and Apache 2.0 License
4// (found in the LICENSE.Apache file in the root directory).
7c673cae
FG
5//
6#include "util/file_reader_writer.h"
7#include <algorithm>
8#include <vector>
9#include "util/random.h"
10#include "util/testharness.h"
11#include "util/testutil.h"
12
13namespace rocksdb {
14
15class WritableFileWriterTest : public testing::Test {};
16
17const uint32_t kMb = 1 << 20;
18
19TEST_F(WritableFileWriterTest, RangeSync) {
20 class FakeWF : public WritableFile {
21 public:
22 explicit FakeWF() : size_(0), last_synced_(0) {}
494da23a 23 ~FakeWF() override {}
7c673cae
FG
24
25 Status Append(const Slice& data) override {
26 size_ += data.size();
27 return Status::OK();
28 }
494da23a 29 Status Truncate(uint64_t /*size*/) override { return Status::OK(); }
7c673cae
FG
30 Status Close() override {
31 EXPECT_GE(size_, last_synced_ + kMb);
32 EXPECT_LT(size_, last_synced_ + 2 * kMb);
33 // Make sure random writes generated enough writes.
34 EXPECT_GT(size_, 10 * kMb);
35 return Status::OK();
36 }
37 Status Flush() override { return Status::OK(); }
38 Status Sync() override { return Status::OK(); }
39 Status Fsync() override { return Status::OK(); }
11fdf7f2 40 void SetIOPriority(Env::IOPriority /*pri*/) override {}
7c673cae 41 uint64_t GetFileSize() override { return size_; }
11fdf7f2
TL
42 void GetPreallocationStatus(size_t* /*block_size*/,
43 size_t* /*last_allocated_block*/) override {}
44 size_t GetUniqueId(char* /*id*/, size_t /*max_size*/) const override {
45 return 0;
46 }
47 Status InvalidateCache(size_t /*offset*/, size_t /*length*/) override {
7c673cae
FG
48 return Status::OK();
49 }
50
51 protected:
11fdf7f2
TL
52 Status Allocate(uint64_t /*offset*/, uint64_t /*len*/) override {
53 return Status::OK();
54 }
7c673cae
FG
55 Status RangeSync(uint64_t offset, uint64_t nbytes) override {
56 EXPECT_EQ(offset % 4096, 0u);
57 EXPECT_EQ(nbytes % 4096, 0u);
58
59 EXPECT_EQ(offset, last_synced_);
60 last_synced_ = offset + nbytes;
61 EXPECT_GE(size_, last_synced_ + kMb);
62 if (size_ > 2 * kMb) {
63 EXPECT_LT(size_, last_synced_ + 2 * kMb);
64 }
65 return Status::OK();
66 }
67
68 uint64_t size_;
69 uint64_t last_synced_;
70 };
71
72 EnvOptions env_options;
73 env_options.bytes_per_sync = kMb;
494da23a
TL
74 std::unique_ptr<FakeWF> wf(new FakeWF);
75 std::unique_ptr<WritableFileWriter> writer(
11fdf7f2 76 new WritableFileWriter(std::move(wf), "" /* don't care */, env_options));
7c673cae
FG
77 Random r(301);
78 std::unique_ptr<char[]> large_buf(new char[10 * kMb]);
79 for (int i = 0; i < 1000; i++) {
80 int skew_limit = (i < 700) ? 10 : 15;
81 uint32_t num = r.Skewed(skew_limit) * 100 + r.Uniform(100);
82 writer->Append(Slice(large_buf.get(), num));
83
84 // Flush in a chance of 1/10.
85 if (r.Uniform(10) == 0) {
86 writer->Flush();
87 }
88 }
89 writer->Close();
90}
91
11fdf7f2
TL
92TEST_F(WritableFileWriterTest, IncrementalBuffer) {
93 class FakeWF : public WritableFile {
94 public:
95 explicit FakeWF(std::string* _file_data, bool _use_direct_io,
96 bool _no_flush)
97 : file_data_(_file_data),
98 use_direct_io_(_use_direct_io),
99 no_flush_(_no_flush) {}
494da23a 100 ~FakeWF() override {}
11fdf7f2
TL
101
102 Status Append(const Slice& data) override {
103 file_data_->append(data.data(), data.size());
104 size_ += data.size();
105 return Status::OK();
106 }
107 Status PositionedAppend(const Slice& data, uint64_t pos) override {
108 EXPECT_TRUE(pos % 512 == 0);
109 EXPECT_TRUE(data.size() % 512 == 0);
110 file_data_->resize(pos);
111 file_data_->append(data.data(), data.size());
112 size_ += data.size();
113 return Status::OK();
114 }
115
494da23a 116 Status Truncate(uint64_t size) override {
11fdf7f2
TL
117 file_data_->resize(size);
118 return Status::OK();
119 }
120 Status Close() override { return Status::OK(); }
121 Status Flush() override { return Status::OK(); }
122 Status Sync() override { return Status::OK(); }
123 Status Fsync() override { return Status::OK(); }
124 void SetIOPriority(Env::IOPriority /*pri*/) override {}
125 uint64_t GetFileSize() override { return size_; }
126 void GetPreallocationStatus(size_t* /*block_size*/,
127 size_t* /*last_allocated_block*/) override {}
128 size_t GetUniqueId(char* /*id*/, size_t /*max_size*/) const override {
129 return 0;
130 }
131 Status InvalidateCache(size_t /*offset*/, size_t /*length*/) override {
132 return Status::OK();
133 }
134 bool use_direct_io() const override { return use_direct_io_; }
135
136 std::string* file_data_;
137 bool use_direct_io_;
138 bool no_flush_;
139 size_t size_ = 0;
140 };
141
142 Random r(301);
143 const int kNumAttempts = 50;
144 for (int attempt = 0; attempt < kNumAttempts; attempt++) {
145 bool no_flush = (attempt % 3 == 0);
146 EnvOptions env_options;
147 env_options.writable_file_max_buffer_size =
148 (attempt < kNumAttempts / 2) ? 512 * 1024 : 700 * 1024;
149 std::string actual;
494da23a 150 std::unique_ptr<FakeWF> wf(new FakeWF(&actual,
11fdf7f2 151#ifndef ROCKSDB_LITE
494da23a 152 attempt % 2 == 1,
11fdf7f2 153#else
494da23a 154 false,
11fdf7f2 155#endif
494da23a
TL
156 no_flush));
157 std::unique_ptr<WritableFileWriter> writer(new WritableFileWriter(
11fdf7f2
TL
158 std::move(wf), "" /* don't care */, env_options));
159
160 std::string target;
161 for (int i = 0; i < 20; i++) {
162 uint32_t num = r.Skewed(16) * 100 + r.Uniform(100);
163 std::string random_string;
164 test::RandomString(&r, num, &random_string);
165 writer->Append(Slice(random_string.c_str(), num));
166 target.append(random_string.c_str(), num);
167
168 // In some attempts, flush in a chance of 1/10.
169 if (!no_flush && r.Uniform(10) == 0) {
170 writer->Flush();
171 }
172 }
173 writer->Flush();
174 writer->Close();
175 ASSERT_EQ(target.size(), actual.size());
176 ASSERT_EQ(target, actual);
177 }
178}
179
7c673cae
FG
180#ifndef ROCKSDB_LITE
181TEST_F(WritableFileWriterTest, AppendStatusReturn) {
182 class FakeWF : public WritableFile {
183 public:
184 explicit FakeWF() : use_direct_io_(false), io_error_(false) {}
185
494da23a 186 bool use_direct_io() const override { return use_direct_io_; }
11fdf7f2 187 Status Append(const Slice& /*data*/) override {
7c673cae
FG
188 if (io_error_) {
189 return Status::IOError("Fake IO error");
190 }
191 return Status::OK();
192 }
11fdf7f2 193 Status PositionedAppend(const Slice& /*data*/, uint64_t) override {
7c673cae
FG
194 if (io_error_) {
195 return Status::IOError("Fake IO error");
196 }
197 return Status::OK();
198 }
199 Status Close() override { return Status::OK(); }
200 Status Flush() override { return Status::OK(); }
201 Status Sync() override { return Status::OK(); }
202 void Setuse_direct_io(bool val) { use_direct_io_ = val; }
203 void SetIOError(bool val) { io_error_ = val; }
204
205 protected:
206 bool use_direct_io_;
207 bool io_error_;
208 };
494da23a 209 std::unique_ptr<FakeWF> wf(new FakeWF());
7c673cae 210 wf->Setuse_direct_io(true);
494da23a 211 std::unique_ptr<WritableFileWriter> writer(
11fdf7f2 212 new WritableFileWriter(std::move(wf), "" /* don't care */, EnvOptions()));
7c673cae
FG
213
214 ASSERT_OK(writer->Append(std::string(2 * kMb, 'a')));
215
216 // Next call to WritableFile::Append() should fail
217 dynamic_cast<FakeWF*>(writer->writable_file())->SetIOError(true);
218 ASSERT_NOK(writer->Append(std::string(2 * kMb, 'b')));
219}
220#endif
221
222class ReadaheadRandomAccessFileTest
223 : public testing::Test,
224 public testing::WithParamInterface<size_t> {
225 public:
226 static std::vector<size_t> GetReadaheadSizeList() {
227 return {1lu << 12, 1lu << 16};
228 }
494da23a 229 void SetUp() override {
7c673cae
FG
230 readahead_size_ = GetParam();
231 scratch_.reset(new char[2 * readahead_size_]);
232 ResetSourceStr();
233 }
234 ReadaheadRandomAccessFileTest() : control_contents_() {}
235 std::string Read(uint64_t offset, size_t n) {
236 Slice result;
237 test_read_holder_->Read(offset, n, &result, scratch_.get());
238 return std::string(result.data(), result.size());
239 }
240 void ResetSourceStr(const std::string& str = "") {
11fdf7f2
TL
241 auto write_holder =
242 std::unique_ptr<WritableFileWriter>(test::GetWritableFileWriter(
243 new test::StringSink(&control_contents_), "" /* don't care */));
7c673cae
FG
244 write_holder->Append(Slice(str));
245 write_holder->Flush();
246 auto read_holder = std::unique_ptr<RandomAccessFile>(
247 new test::StringSource(control_contents_));
248 test_read_holder_ =
249 NewReadaheadRandomAccessFile(std::move(read_holder), readahead_size_);
250 }
251 size_t GetReadaheadSize() const { return readahead_size_; }
252
253 private:
254 size_t readahead_size_;
255 Slice control_contents_;
256 std::unique_ptr<RandomAccessFile> test_read_holder_;
257 std::unique_ptr<char[]> scratch_;
258};
259
260TEST_P(ReadaheadRandomAccessFileTest, EmptySourceStrTest) {
261 ASSERT_EQ("", Read(0, 1));
262 ASSERT_EQ("", Read(0, 0));
263 ASSERT_EQ("", Read(13, 13));
264}
265
266TEST_P(ReadaheadRandomAccessFileTest, SourceStrLenLessThanReadaheadSizeTest) {
267 std::string str = "abcdefghijklmnopqrs";
268 ResetSourceStr(str);
269 ASSERT_EQ(str.substr(3, 4), Read(3, 4));
270 ASSERT_EQ(str.substr(0, 3), Read(0, 3));
271 ASSERT_EQ(str, Read(0, str.size()));
272 ASSERT_EQ(str.substr(7, std::min(static_cast<int>(str.size()) - 7, 30)),
273 Read(7, 30));
274 ASSERT_EQ("", Read(100, 100));
275}
276
277TEST_P(ReadaheadRandomAccessFileTest,
278 SourceStrLenCanBeGreaterThanReadaheadSizeTest) {
279 Random rng(42);
280 for (int k = 0; k < 100; ++k) {
281 size_t strLen = k * GetReadaheadSize() +
282 rng.Uniform(static_cast<int>(GetReadaheadSize()));
283 std::string str =
284 test::RandomHumanReadableString(&rng, static_cast<int>(strLen));
285 ResetSourceStr(str);
286 for (int test = 1; test <= 100; ++test) {
287 size_t offset = rng.Uniform(static_cast<int>(strLen));
288 size_t n = rng.Uniform(static_cast<int>(GetReadaheadSize()));
289 ASSERT_EQ(str.substr(offset, std::min(n, str.size() - offset)),
290 Read(offset, n));
291 }
292 }
293}
294
295TEST_P(ReadaheadRandomAccessFileTest, NExceedReadaheadTest) {
296 Random rng(7);
297 size_t strLen = 4 * GetReadaheadSize() +
298 rng.Uniform(static_cast<int>(GetReadaheadSize()));
299 std::string str =
300 test::RandomHumanReadableString(&rng, static_cast<int>(strLen));
301 ResetSourceStr(str);
302 for (int test = 1; test <= 100; ++test) {
303 size_t offset = rng.Uniform(static_cast<int>(strLen));
304 size_t n =
305 GetReadaheadSize() + rng.Uniform(static_cast<int>(GetReadaheadSize()));
306 ASSERT_EQ(str.substr(offset, std::min(n, str.size() - offset)),
307 Read(offset, n));
308 }
309}
310
311INSTANTIATE_TEST_CASE_P(
312 EmptySourceStrTest, ReadaheadRandomAccessFileTest,
313 ::testing::ValuesIn(ReadaheadRandomAccessFileTest::GetReadaheadSizeList()));
314INSTANTIATE_TEST_CASE_P(
315 SourceStrLenLessThanReadaheadSizeTest, ReadaheadRandomAccessFileTest,
316 ::testing::ValuesIn(ReadaheadRandomAccessFileTest::GetReadaheadSizeList()));
317INSTANTIATE_TEST_CASE_P(
318 SourceStrLenCanBeGreaterThanReadaheadSizeTest,
319 ReadaheadRandomAccessFileTest,
320 ::testing::ValuesIn(ReadaheadRandomAccessFileTest::GetReadaheadSizeList()));
321INSTANTIATE_TEST_CASE_P(
322 NExceedReadaheadTest, ReadaheadRandomAccessFileTest,
323 ::testing::ValuesIn(ReadaheadRandomAccessFileTest::GetReadaheadSizeList()));
324
325} // namespace rocksdb
326
327int main(int argc, char** argv) {
328 ::testing::InitGoogleTest(&argc, argv);
329 return RUN_ALL_TESTS();
330}