]>
Commit | Line | Data |
---|---|---|
7c673cae | 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. |
11fdf7f2 TL |
2 | // This source code is licensed under both the GPLv2 (found in the |
3 | // COPYING file in the root directory) and Apache 2.0 License | |
4 | // (found in the LICENSE.Apache file in the root directory). | |
7c673cae FG |
5 | // |
6 | #include "util/file_reader_writer.h" | |
7 | #include <algorithm> | |
8 | #include <vector> | |
9 | #include "util/random.h" | |
10 | #include "util/testharness.h" | |
11 | #include "util/testutil.h" | |
12 | ||
13 | namespace rocksdb { | |
14 | ||
15 | class WritableFileWriterTest : public testing::Test {}; | |
16 | ||
17 | const uint32_t kMb = 1 << 20; | |
18 | ||
19 | TEST_F(WritableFileWriterTest, RangeSync) { | |
20 | class FakeWF : public WritableFile { | |
21 | public: | |
22 | explicit FakeWF() : size_(0), last_synced_(0) {} | |
494da23a | 23 | ~FakeWF() override {} |
7c673cae FG |
24 | |
25 | Status Append(const Slice& data) override { | |
26 | size_ += data.size(); | |
27 | return Status::OK(); | |
28 | } | |
494da23a | 29 | Status Truncate(uint64_t /*size*/) override { return Status::OK(); } |
7c673cae FG |
30 | Status Close() override { |
31 | EXPECT_GE(size_, last_synced_ + kMb); | |
32 | EXPECT_LT(size_, last_synced_ + 2 * kMb); | |
33 | // Make sure random writes generated enough writes. | |
34 | EXPECT_GT(size_, 10 * kMb); | |
35 | return Status::OK(); | |
36 | } | |
37 | Status Flush() override { return Status::OK(); } | |
38 | Status Sync() override { return Status::OK(); } | |
39 | Status Fsync() override { return Status::OK(); } | |
11fdf7f2 | 40 | void SetIOPriority(Env::IOPriority /*pri*/) override {} |
7c673cae | 41 | uint64_t GetFileSize() override { return size_; } |
11fdf7f2 TL |
42 | void GetPreallocationStatus(size_t* /*block_size*/, |
43 | size_t* /*last_allocated_block*/) override {} | |
44 | size_t GetUniqueId(char* /*id*/, size_t /*max_size*/) const override { | |
45 | return 0; | |
46 | } | |
47 | Status InvalidateCache(size_t /*offset*/, size_t /*length*/) override { | |
7c673cae FG |
48 | return Status::OK(); |
49 | } | |
50 | ||
51 | protected: | |
11fdf7f2 TL |
52 | Status Allocate(uint64_t /*offset*/, uint64_t /*len*/) override { |
53 | return Status::OK(); | |
54 | } | |
7c673cae FG |
55 | Status RangeSync(uint64_t offset, uint64_t nbytes) override { |
56 | EXPECT_EQ(offset % 4096, 0u); | |
57 | EXPECT_EQ(nbytes % 4096, 0u); | |
58 | ||
59 | EXPECT_EQ(offset, last_synced_); | |
60 | last_synced_ = offset + nbytes; | |
61 | EXPECT_GE(size_, last_synced_ + kMb); | |
62 | if (size_ > 2 * kMb) { | |
63 | EXPECT_LT(size_, last_synced_ + 2 * kMb); | |
64 | } | |
65 | return Status::OK(); | |
66 | } | |
67 | ||
68 | uint64_t size_; | |
69 | uint64_t last_synced_; | |
70 | }; | |
71 | ||
72 | EnvOptions env_options; | |
73 | env_options.bytes_per_sync = kMb; | |
494da23a TL |
74 | std::unique_ptr<FakeWF> wf(new FakeWF); |
75 | std::unique_ptr<WritableFileWriter> writer( | |
11fdf7f2 | 76 | new WritableFileWriter(std::move(wf), "" /* don't care */, env_options)); |
7c673cae FG |
77 | Random r(301); |
78 | std::unique_ptr<char[]> large_buf(new char[10 * kMb]); | |
79 | for (int i = 0; i < 1000; i++) { | |
80 | int skew_limit = (i < 700) ? 10 : 15; | |
81 | uint32_t num = r.Skewed(skew_limit) * 100 + r.Uniform(100); | |
82 | writer->Append(Slice(large_buf.get(), num)); | |
83 | ||
84 | // Flush in a chance of 1/10. | |
85 | if (r.Uniform(10) == 0) { | |
86 | writer->Flush(); | |
87 | } | |
88 | } | |
89 | writer->Close(); | |
90 | } | |
91 | ||
11fdf7f2 TL |
92 | TEST_F(WritableFileWriterTest, IncrementalBuffer) { |
93 | class FakeWF : public WritableFile { | |
94 | public: | |
95 | explicit FakeWF(std::string* _file_data, bool _use_direct_io, | |
96 | bool _no_flush) | |
97 | : file_data_(_file_data), | |
98 | use_direct_io_(_use_direct_io), | |
99 | no_flush_(_no_flush) {} | |
494da23a | 100 | ~FakeWF() override {} |
11fdf7f2 TL |
101 | |
102 | Status Append(const Slice& data) override { | |
103 | file_data_->append(data.data(), data.size()); | |
104 | size_ += data.size(); | |
105 | return Status::OK(); | |
106 | } | |
107 | Status PositionedAppend(const Slice& data, uint64_t pos) override { | |
108 | EXPECT_TRUE(pos % 512 == 0); | |
109 | EXPECT_TRUE(data.size() % 512 == 0); | |
110 | file_data_->resize(pos); | |
111 | file_data_->append(data.data(), data.size()); | |
112 | size_ += data.size(); | |
113 | return Status::OK(); | |
114 | } | |
115 | ||
494da23a | 116 | Status Truncate(uint64_t size) override { |
11fdf7f2 TL |
117 | file_data_->resize(size); |
118 | return Status::OK(); | |
119 | } | |
120 | Status Close() override { return Status::OK(); } | |
121 | Status Flush() override { return Status::OK(); } | |
122 | Status Sync() override { return Status::OK(); } | |
123 | Status Fsync() override { return Status::OK(); } | |
124 | void SetIOPriority(Env::IOPriority /*pri*/) override {} | |
125 | uint64_t GetFileSize() override { return size_; } | |
126 | void GetPreallocationStatus(size_t* /*block_size*/, | |
127 | size_t* /*last_allocated_block*/) override {} | |
128 | size_t GetUniqueId(char* /*id*/, size_t /*max_size*/) const override { | |
129 | return 0; | |
130 | } | |
131 | Status InvalidateCache(size_t /*offset*/, size_t /*length*/) override { | |
132 | return Status::OK(); | |
133 | } | |
134 | bool use_direct_io() const override { return use_direct_io_; } | |
135 | ||
136 | std::string* file_data_; | |
137 | bool use_direct_io_; | |
138 | bool no_flush_; | |
139 | size_t size_ = 0; | |
140 | }; | |
141 | ||
142 | Random r(301); | |
143 | const int kNumAttempts = 50; | |
144 | for (int attempt = 0; attempt < kNumAttempts; attempt++) { | |
145 | bool no_flush = (attempt % 3 == 0); | |
146 | EnvOptions env_options; | |
147 | env_options.writable_file_max_buffer_size = | |
148 | (attempt < kNumAttempts / 2) ? 512 * 1024 : 700 * 1024; | |
149 | std::string actual; | |
494da23a | 150 | std::unique_ptr<FakeWF> wf(new FakeWF(&actual, |
11fdf7f2 | 151 | #ifndef ROCKSDB_LITE |
494da23a | 152 | attempt % 2 == 1, |
11fdf7f2 | 153 | #else |
494da23a | 154 | false, |
11fdf7f2 | 155 | #endif |
494da23a TL |
156 | no_flush)); |
157 | std::unique_ptr<WritableFileWriter> writer(new WritableFileWriter( | |
11fdf7f2 TL |
158 | std::move(wf), "" /* don't care */, env_options)); |
159 | ||
160 | std::string target; | |
161 | for (int i = 0; i < 20; i++) { | |
162 | uint32_t num = r.Skewed(16) * 100 + r.Uniform(100); | |
163 | std::string random_string; | |
164 | test::RandomString(&r, num, &random_string); | |
165 | writer->Append(Slice(random_string.c_str(), num)); | |
166 | target.append(random_string.c_str(), num); | |
167 | ||
168 | // In some attempts, flush in a chance of 1/10. | |
169 | if (!no_flush && r.Uniform(10) == 0) { | |
170 | writer->Flush(); | |
171 | } | |
172 | } | |
173 | writer->Flush(); | |
174 | writer->Close(); | |
175 | ASSERT_EQ(target.size(), actual.size()); | |
176 | ASSERT_EQ(target, actual); | |
177 | } | |
178 | } | |
179 | ||
7c673cae FG |
180 | #ifndef ROCKSDB_LITE |
181 | TEST_F(WritableFileWriterTest, AppendStatusReturn) { | |
182 | class FakeWF : public WritableFile { | |
183 | public: | |
184 | explicit FakeWF() : use_direct_io_(false), io_error_(false) {} | |
185 | ||
494da23a | 186 | bool use_direct_io() const override { return use_direct_io_; } |
11fdf7f2 | 187 | Status Append(const Slice& /*data*/) override { |
7c673cae FG |
188 | if (io_error_) { |
189 | return Status::IOError("Fake IO error"); | |
190 | } | |
191 | return Status::OK(); | |
192 | } | |
11fdf7f2 | 193 | Status PositionedAppend(const Slice& /*data*/, uint64_t) override { |
7c673cae FG |
194 | if (io_error_) { |
195 | return Status::IOError("Fake IO error"); | |
196 | } | |
197 | return Status::OK(); | |
198 | } | |
199 | Status Close() override { return Status::OK(); } | |
200 | Status Flush() override { return Status::OK(); } | |
201 | Status Sync() override { return Status::OK(); } | |
202 | void Setuse_direct_io(bool val) { use_direct_io_ = val; } | |
203 | void SetIOError(bool val) { io_error_ = val; } | |
204 | ||
205 | protected: | |
206 | bool use_direct_io_; | |
207 | bool io_error_; | |
208 | }; | |
494da23a | 209 | std::unique_ptr<FakeWF> wf(new FakeWF()); |
7c673cae | 210 | wf->Setuse_direct_io(true); |
494da23a | 211 | std::unique_ptr<WritableFileWriter> writer( |
11fdf7f2 | 212 | new WritableFileWriter(std::move(wf), "" /* don't care */, EnvOptions())); |
7c673cae FG |
213 | |
214 | ASSERT_OK(writer->Append(std::string(2 * kMb, 'a'))); | |
215 | ||
216 | // Next call to WritableFile::Append() should fail | |
217 | dynamic_cast<FakeWF*>(writer->writable_file())->SetIOError(true); | |
218 | ASSERT_NOK(writer->Append(std::string(2 * kMb, 'b'))); | |
219 | } | |
220 | #endif | |
221 | ||
222 | class ReadaheadRandomAccessFileTest | |
223 | : public testing::Test, | |
224 | public testing::WithParamInterface<size_t> { | |
225 | public: | |
226 | static std::vector<size_t> GetReadaheadSizeList() { | |
227 | return {1lu << 12, 1lu << 16}; | |
228 | } | |
494da23a | 229 | void SetUp() override { |
7c673cae FG |
230 | readahead_size_ = GetParam(); |
231 | scratch_.reset(new char[2 * readahead_size_]); | |
232 | ResetSourceStr(); | |
233 | } | |
234 | ReadaheadRandomAccessFileTest() : control_contents_() {} | |
235 | std::string Read(uint64_t offset, size_t n) { | |
236 | Slice result; | |
237 | test_read_holder_->Read(offset, n, &result, scratch_.get()); | |
238 | return std::string(result.data(), result.size()); | |
239 | } | |
240 | void ResetSourceStr(const std::string& str = "") { | |
11fdf7f2 TL |
241 | auto write_holder = |
242 | std::unique_ptr<WritableFileWriter>(test::GetWritableFileWriter( | |
243 | new test::StringSink(&control_contents_), "" /* don't care */)); | |
7c673cae FG |
244 | write_holder->Append(Slice(str)); |
245 | write_holder->Flush(); | |
246 | auto read_holder = std::unique_ptr<RandomAccessFile>( | |
247 | new test::StringSource(control_contents_)); | |
248 | test_read_holder_ = | |
249 | NewReadaheadRandomAccessFile(std::move(read_holder), readahead_size_); | |
250 | } | |
251 | size_t GetReadaheadSize() const { return readahead_size_; } | |
252 | ||
253 | private: | |
254 | size_t readahead_size_; | |
255 | Slice control_contents_; | |
256 | std::unique_ptr<RandomAccessFile> test_read_holder_; | |
257 | std::unique_ptr<char[]> scratch_; | |
258 | }; | |
259 | ||
260 | TEST_P(ReadaheadRandomAccessFileTest, EmptySourceStrTest) { | |
261 | ASSERT_EQ("", Read(0, 1)); | |
262 | ASSERT_EQ("", Read(0, 0)); | |
263 | ASSERT_EQ("", Read(13, 13)); | |
264 | } | |
265 | ||
266 | TEST_P(ReadaheadRandomAccessFileTest, SourceStrLenLessThanReadaheadSizeTest) { | |
267 | std::string str = "abcdefghijklmnopqrs"; | |
268 | ResetSourceStr(str); | |
269 | ASSERT_EQ(str.substr(3, 4), Read(3, 4)); | |
270 | ASSERT_EQ(str.substr(0, 3), Read(0, 3)); | |
271 | ASSERT_EQ(str, Read(0, str.size())); | |
272 | ASSERT_EQ(str.substr(7, std::min(static_cast<int>(str.size()) - 7, 30)), | |
273 | Read(7, 30)); | |
274 | ASSERT_EQ("", Read(100, 100)); | |
275 | } | |
276 | ||
277 | TEST_P(ReadaheadRandomAccessFileTest, | |
278 | SourceStrLenCanBeGreaterThanReadaheadSizeTest) { | |
279 | Random rng(42); | |
280 | for (int k = 0; k < 100; ++k) { | |
281 | size_t strLen = k * GetReadaheadSize() + | |
282 | rng.Uniform(static_cast<int>(GetReadaheadSize())); | |
283 | std::string str = | |
284 | test::RandomHumanReadableString(&rng, static_cast<int>(strLen)); | |
285 | ResetSourceStr(str); | |
286 | for (int test = 1; test <= 100; ++test) { | |
287 | size_t offset = rng.Uniform(static_cast<int>(strLen)); | |
288 | size_t n = rng.Uniform(static_cast<int>(GetReadaheadSize())); | |
289 | ASSERT_EQ(str.substr(offset, std::min(n, str.size() - offset)), | |
290 | Read(offset, n)); | |
291 | } | |
292 | } | |
293 | } | |
294 | ||
295 | TEST_P(ReadaheadRandomAccessFileTest, NExceedReadaheadTest) { | |
296 | Random rng(7); | |
297 | size_t strLen = 4 * GetReadaheadSize() + | |
298 | rng.Uniform(static_cast<int>(GetReadaheadSize())); | |
299 | std::string str = | |
300 | test::RandomHumanReadableString(&rng, static_cast<int>(strLen)); | |
301 | ResetSourceStr(str); | |
302 | for (int test = 1; test <= 100; ++test) { | |
303 | size_t offset = rng.Uniform(static_cast<int>(strLen)); | |
304 | size_t n = | |
305 | GetReadaheadSize() + rng.Uniform(static_cast<int>(GetReadaheadSize())); | |
306 | ASSERT_EQ(str.substr(offset, std::min(n, str.size() - offset)), | |
307 | Read(offset, n)); | |
308 | } | |
309 | } | |
310 | ||
311 | INSTANTIATE_TEST_CASE_P( | |
312 | EmptySourceStrTest, ReadaheadRandomAccessFileTest, | |
313 | ::testing::ValuesIn(ReadaheadRandomAccessFileTest::GetReadaheadSizeList())); | |
314 | INSTANTIATE_TEST_CASE_P( | |
315 | SourceStrLenLessThanReadaheadSizeTest, ReadaheadRandomAccessFileTest, | |
316 | ::testing::ValuesIn(ReadaheadRandomAccessFileTest::GetReadaheadSizeList())); | |
317 | INSTANTIATE_TEST_CASE_P( | |
318 | SourceStrLenCanBeGreaterThanReadaheadSizeTest, | |
319 | ReadaheadRandomAccessFileTest, | |
320 | ::testing::ValuesIn(ReadaheadRandomAccessFileTest::GetReadaheadSizeList())); | |
321 | INSTANTIATE_TEST_CASE_P( | |
322 | NExceedReadaheadTest, ReadaheadRandomAccessFileTest, | |
323 | ::testing::ValuesIn(ReadaheadRandomAccessFileTest::GetReadaheadSizeList())); | |
324 | ||
325 | } // namespace rocksdb | |
326 | ||
327 | int main(int argc, char** argv) { | |
328 | ::testing::InitGoogleTest(&argc, argv); | |
329 | return RUN_ALL_TESTS(); | |
330 | } |