]>
Commit | Line | Data |
---|---|---|
7c673cae | 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. |
11fdf7f2 TL |
2 | // This source code is licensed under both the GPLv2 (found in the |
3 | // COPYING file in the root directory) and Apache 2.0 License | |
4 | // (found in the LICENSE.Apache file in the root directory). | |
7c673cae FG |
5 | |
6 | #include <functional> | |
7 | ||
8 | #include "db/db_test_util.h" | |
9 | #include "port/port.h" | |
10 | #include "port/stack_trace.h" | |
11 | #include "rocksdb/sst_file_writer.h" | |
12 | #include "util/testutil.h" | |
13 | ||
14 | namespace rocksdb { | |
15 | ||
16 | #ifndef ROCKSDB_LITE | |
17 | class ExternalSSTFileBasicTest : public DBTestBase { | |
18 | public: | |
19 | ExternalSSTFileBasicTest() : DBTestBase("/external_sst_file_test") { | |
20 | sst_files_dir_ = dbname_ + "/sst_files/"; | |
21 | DestroyAndRecreateExternalSSTFilesDir(); | |
22 | } | |
23 | ||
24 | void DestroyAndRecreateExternalSSTFilesDir() { | |
25 | test::DestroyDir(env_, sst_files_dir_); | |
26 | env_->CreateDir(sst_files_dir_); | |
27 | } | |
28 | ||
29 | Status DeprecatedAddFile(const std::vector<std::string>& files, | |
30 | bool move_files = false, | |
31 | bool skip_snapshot_check = false) { | |
32 | IngestExternalFileOptions opts; | |
33 | opts.move_files = move_files; | |
34 | opts.snapshot_consistency = !skip_snapshot_check; | |
35 | opts.allow_global_seqno = false; | |
36 | opts.allow_blocking_flush = false; | |
37 | return db_->IngestExternalFile(files, opts); | |
38 | } | |
39 | ||
40 | Status GenerateAndAddExternalFile( | |
11fdf7f2 TL |
41 | const Options options, std::vector<int> keys, |
42 | const std::vector<ValueType>& value_types, | |
43 | std::vector<std::pair<int, int>> range_deletions, int file_id, | |
7c673cae | 44 | std::map<std::string, std::string>* true_data) { |
11fdf7f2 | 45 | assert(value_types.size() == 1 || keys.size() == value_types.size()); |
7c673cae FG |
46 | std::string file_path = sst_files_dir_ + ToString(file_id); |
47 | SstFileWriter sst_file_writer(EnvOptions(), options); | |
48 | ||
49 | Status s = sst_file_writer.Open(file_path); | |
50 | if (!s.ok()) { | |
51 | return s; | |
52 | } | |
11fdf7f2 TL |
53 | for (size_t i = 0; i < range_deletions.size(); i++) { |
54 | // Account for the effect of range deletions on true_data before | |
55 | // all point operators, even though sst_file_writer.DeleteRange | |
56 | // must be called before other sst_file_writer methods. This is | |
57 | // because point writes take precedence over range deletions | |
58 | // in the same ingested sst. | |
59 | std::string start_key = Key(range_deletions[i].first); | |
60 | std::string end_key = Key(range_deletions[i].second); | |
61 | s = sst_file_writer.DeleteRange(start_key, end_key); | |
62 | if (!s.ok()) { | |
63 | sst_file_writer.Finish(); | |
64 | return s; | |
65 | } | |
66 | auto start_key_it = true_data->find(start_key); | |
67 | if (start_key_it == true_data->end()) { | |
68 | start_key_it = true_data->upper_bound(start_key); | |
69 | } | |
70 | auto end_key_it = true_data->find(end_key); | |
71 | if (end_key_it == true_data->end()) { | |
72 | end_key_it = true_data->upper_bound(end_key); | |
73 | } | |
74 | true_data->erase(start_key_it, end_key_it); | |
75 | } | |
76 | for (size_t i = 0; i < keys.size(); i++) { | |
77 | std::string key = Key(keys[i]); | |
78 | std::string value = Key(keys[i]) + ToString(file_id); | |
79 | ValueType value_type = | |
80 | (value_types.size() == 1 ? value_types[0] : value_types[i]); | |
81 | switch (value_type) { | |
82 | case ValueType::kTypeValue: | |
83 | s = sst_file_writer.Put(key, value); | |
84 | (*true_data)[key] = value; | |
85 | break; | |
86 | case ValueType::kTypeMerge: | |
87 | s = sst_file_writer.Merge(key, value); | |
88 | // we only use TestPutOperator in this test | |
89 | (*true_data)[key] = value; | |
90 | break; | |
91 | case ValueType::kTypeDeletion: | |
92 | s = sst_file_writer.Delete(key); | |
93 | true_data->erase(key); | |
94 | break; | |
95 | default: | |
96 | return Status::InvalidArgument("Value type is not supported"); | |
97 | } | |
7c673cae FG |
98 | if (!s.ok()) { |
99 | sst_file_writer.Finish(); | |
100 | return s; | |
101 | } | |
102 | } | |
103 | s = sst_file_writer.Finish(); | |
104 | ||
105 | if (s.ok()) { | |
106 | IngestExternalFileOptions ifo; | |
107 | ifo.allow_global_seqno = true; | |
108 | s = db_->IngestExternalFile({file_path}, ifo); | |
109 | } | |
7c673cae FG |
110 | return s; |
111 | } | |
112 | ||
11fdf7f2 TL |
113 | Status GenerateAndAddExternalFile( |
114 | const Options options, std::vector<int> keys, | |
115 | const std::vector<ValueType>& value_types, int file_id, | |
116 | std::map<std::string, std::string>* true_data) { | |
117 | return GenerateAndAddExternalFile(options, keys, value_types, {}, file_id, | |
118 | true_data); | |
119 | } | |
120 | ||
121 | Status GenerateAndAddExternalFile( | |
122 | const Options options, std::vector<int> keys, const ValueType value_type, | |
123 | int file_id, std::map<std::string, std::string>* true_data) { | |
124 | return GenerateAndAddExternalFile(options, keys, | |
125 | std::vector<ValueType>(1, value_type), | |
126 | file_id, true_data); | |
127 | } | |
128 | ||
7c673cae FG |
129 | ~ExternalSSTFileBasicTest() { test::DestroyDir(env_, sst_files_dir_); } |
130 | ||
131 | protected: | |
132 | std::string sst_files_dir_; | |
133 | }; | |
134 | ||
135 | TEST_F(ExternalSSTFileBasicTest, Basic) { | |
136 | Options options = CurrentOptions(); | |
137 | ||
138 | SstFileWriter sst_file_writer(EnvOptions(), options); | |
139 | ||
140 | // Current file size should be 0 after sst_file_writer init and before open a | |
141 | // file. | |
142 | ASSERT_EQ(sst_file_writer.FileSize(), 0); | |
143 | ||
144 | // file1.sst (0 => 99) | |
145 | std::string file1 = sst_files_dir_ + "file1.sst"; | |
146 | ASSERT_OK(sst_file_writer.Open(file1)); | |
147 | for (int k = 0; k < 100; k++) { | |
11fdf7f2 | 148 | ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val")); |
7c673cae FG |
149 | } |
150 | ExternalSstFileInfo file1_info; | |
151 | Status s = sst_file_writer.Finish(&file1_info); | |
152 | ASSERT_TRUE(s.ok()) << s.ToString(); | |
153 | ||
154 | // Current file size should be non-zero after success write. | |
155 | ASSERT_GT(sst_file_writer.FileSize(), 0); | |
156 | ||
157 | ASSERT_EQ(file1_info.file_path, file1); | |
158 | ASSERT_EQ(file1_info.num_entries, 100); | |
159 | ASSERT_EQ(file1_info.smallest_key, Key(0)); | |
160 | ASSERT_EQ(file1_info.largest_key, Key(99)); | |
11fdf7f2 TL |
161 | ASSERT_EQ(file1_info.num_range_del_entries, 0); |
162 | ASSERT_EQ(file1_info.smallest_range_del_key, ""); | |
163 | ASSERT_EQ(file1_info.largest_range_del_key, ""); | |
7c673cae | 164 | // sst_file_writer already finished, cannot add this value |
11fdf7f2 TL |
165 | s = sst_file_writer.Put(Key(100), "bad_val"); |
166 | ASSERT_FALSE(s.ok()) << s.ToString(); | |
167 | s = sst_file_writer.DeleteRange(Key(100), Key(200)); | |
7c673cae FG |
168 | ASSERT_FALSE(s.ok()) << s.ToString(); |
169 | ||
170 | DestroyAndReopen(options); | |
171 | // Add file using file path | |
172 | s = DeprecatedAddFile({file1}); | |
173 | ASSERT_TRUE(s.ok()) << s.ToString(); | |
174 | ASSERT_EQ(db_->GetLatestSequenceNumber(), 0U); | |
175 | for (int k = 0; k < 100; k++) { | |
176 | ASSERT_EQ(Get(Key(k)), Key(k) + "_val"); | |
177 | } | |
178 | ||
179 | DestroyAndRecreateExternalSSTFilesDir(); | |
180 | } | |
181 | ||
182 | TEST_F(ExternalSSTFileBasicTest, NoCopy) { | |
183 | Options options = CurrentOptions(); | |
184 | const ImmutableCFOptions ioptions(options); | |
185 | ||
186 | SstFileWriter sst_file_writer(EnvOptions(), options); | |
187 | ||
188 | // file1.sst (0 => 99) | |
189 | std::string file1 = sst_files_dir_ + "file1.sst"; | |
190 | ASSERT_OK(sst_file_writer.Open(file1)); | |
191 | for (int k = 0; k < 100; k++) { | |
11fdf7f2 | 192 | ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val")); |
7c673cae FG |
193 | } |
194 | ExternalSstFileInfo file1_info; | |
195 | Status s = sst_file_writer.Finish(&file1_info); | |
196 | ASSERT_TRUE(s.ok()) << s.ToString(); | |
197 | ASSERT_EQ(file1_info.file_path, file1); | |
198 | ASSERT_EQ(file1_info.num_entries, 100); | |
199 | ASSERT_EQ(file1_info.smallest_key, Key(0)); | |
200 | ASSERT_EQ(file1_info.largest_key, Key(99)); | |
201 | ||
202 | // file2.sst (100 => 299) | |
203 | std::string file2 = sst_files_dir_ + "file2.sst"; | |
204 | ASSERT_OK(sst_file_writer.Open(file2)); | |
205 | for (int k = 100; k < 300; k++) { | |
11fdf7f2 | 206 | ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val")); |
7c673cae FG |
207 | } |
208 | ExternalSstFileInfo file2_info; | |
209 | s = sst_file_writer.Finish(&file2_info); | |
210 | ASSERT_TRUE(s.ok()) << s.ToString(); | |
211 | ASSERT_EQ(file2_info.file_path, file2); | |
212 | ASSERT_EQ(file2_info.num_entries, 200); | |
213 | ASSERT_EQ(file2_info.smallest_key, Key(100)); | |
214 | ASSERT_EQ(file2_info.largest_key, Key(299)); | |
215 | ||
216 | // file3.sst (110 => 124) .. overlap with file2.sst | |
217 | std::string file3 = sst_files_dir_ + "file3.sst"; | |
218 | ASSERT_OK(sst_file_writer.Open(file3)); | |
219 | for (int k = 110; k < 125; k++) { | |
11fdf7f2 | 220 | ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val_overlap")); |
7c673cae FG |
221 | } |
222 | ExternalSstFileInfo file3_info; | |
223 | s = sst_file_writer.Finish(&file3_info); | |
224 | ASSERT_TRUE(s.ok()) << s.ToString(); | |
225 | ASSERT_EQ(file3_info.file_path, file3); | |
226 | ASSERT_EQ(file3_info.num_entries, 15); | |
227 | ASSERT_EQ(file3_info.smallest_key, Key(110)); | |
228 | ASSERT_EQ(file3_info.largest_key, Key(124)); | |
11fdf7f2 | 229 | |
7c673cae FG |
230 | s = DeprecatedAddFile({file1}, true /* move file */); |
231 | ASSERT_TRUE(s.ok()) << s.ToString(); | |
232 | ASSERT_EQ(Status::NotFound(), env_->FileExists(file1)); | |
233 | ||
234 | s = DeprecatedAddFile({file2}, false /* copy file */); | |
235 | ASSERT_TRUE(s.ok()) << s.ToString(); | |
236 | ASSERT_OK(env_->FileExists(file2)); | |
237 | ||
11fdf7f2 TL |
238 | // This file has overlapping values with the existing data |
239 | s = DeprecatedAddFile({file3}, true /* move file */); | |
7c673cae FG |
240 | ASSERT_FALSE(s.ok()) << s.ToString(); |
241 | ASSERT_OK(env_->FileExists(file3)); | |
242 | ||
243 | for (int k = 0; k < 300; k++) { | |
244 | ASSERT_EQ(Get(Key(k)), Key(k) + "_val"); | |
245 | } | |
246 | } | |
247 | ||
248 | TEST_F(ExternalSSTFileBasicTest, IngestFileWithGlobalSeqnoPickedSeqno) { | |
249 | do { | |
250 | Options options = CurrentOptions(); | |
251 | DestroyAndReopen(options); | |
252 | std::map<std::string, std::string> true_data; | |
253 | ||
254 | int file_id = 1; | |
255 | ||
11fdf7f2 TL |
256 | ASSERT_OK(GenerateAndAddExternalFile(options, {1, 2, 3, 4, 5, 6}, |
257 | ValueType::kTypeValue, file_id++, | |
7c673cae | 258 | &true_data)); |
11fdf7f2 | 259 | // File doesn't overwrite any keys, no seqno needed |
7c673cae FG |
260 | ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 0); |
261 | ||
11fdf7f2 TL |
262 | ASSERT_OK(GenerateAndAddExternalFile(options, {10, 11, 12, 13}, |
263 | ValueType::kTypeValue, file_id++, | |
7c673cae | 264 | &true_data)); |
11fdf7f2 | 265 | // File doesn't overwrite any keys, no seqno needed |
7c673cae FG |
266 | ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 0); |
267 | ||
11fdf7f2 TL |
268 | ASSERT_OK(GenerateAndAddExternalFile( |
269 | options, {1, 4, 6}, ValueType::kTypeValue, file_id++, &true_data)); | |
270 | // File overwrites some keys, a seqno will be assigned | |
7c673cae FG |
271 | ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 1); |
272 | ||
11fdf7f2 TL |
273 | ASSERT_OK(GenerateAndAddExternalFile( |
274 | options, {11, 15, 19}, ValueType::kTypeValue, file_id++, &true_data)); | |
275 | // File overwrites some keys, a seqno will be assigned | |
7c673cae FG |
276 | ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 2); |
277 | ||
11fdf7f2 TL |
278 | ASSERT_OK(GenerateAndAddExternalFile( |
279 | options, {120, 130}, ValueType::kTypeValue, file_id++, &true_data)); | |
280 | // File doesn't overwrite any keys, no seqno needed | |
7c673cae FG |
281 | ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 2); |
282 | ||
11fdf7f2 TL |
283 | ASSERT_OK(GenerateAndAddExternalFile( |
284 | options, {1, 130}, ValueType::kTypeValue, file_id++, &true_data)); | |
285 | // File overwrites some keys, a seqno will be assigned | |
7c673cae FG |
286 | ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 3); |
287 | ||
288 | // Write some keys through normal write path | |
289 | for (int i = 0; i < 50; i++) { | |
290 | ASSERT_OK(Put(Key(i), "memtable")); | |
291 | true_data[Key(i)] = "memtable"; | |
292 | } | |
293 | SequenceNumber last_seqno = dbfull()->GetLatestSequenceNumber(); | |
294 | ||
11fdf7f2 TL |
295 | ASSERT_OK(GenerateAndAddExternalFile( |
296 | options, {60, 61, 62}, ValueType::kTypeValue, file_id++, &true_data)); | |
297 | // File doesn't overwrite any keys, no seqno needed | |
7c673cae FG |
298 | ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno); |
299 | ||
11fdf7f2 TL |
300 | ASSERT_OK(GenerateAndAddExternalFile( |
301 | options, {40, 41, 42}, ValueType::kTypeValue, file_id++, &true_data)); | |
302 | // File overwrites some keys, a seqno will be assigned | |
303 | ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 1); | |
304 | ||
305 | ASSERT_OK(GenerateAndAddExternalFile( | |
306 | options, {20, 30, 40}, ValueType::kTypeValue, file_id++, &true_data)); | |
307 | // File overwrites some keys, a seqno will be assigned | |
308 | ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 2); | |
309 | ||
310 | const Snapshot* snapshot = db_->GetSnapshot(); | |
311 | ||
312 | // We will need a seqno for the file regardless if the file overwrite | |
313 | // keys in the DB or not because we have a snapshot | |
314 | ASSERT_OK(GenerateAndAddExternalFile( | |
315 | options, {1000, 1002}, ValueType::kTypeValue, file_id++, &true_data)); | |
316 | // A global seqno will be assigned anyway because of the snapshot | |
317 | ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 3); | |
318 | ||
319 | ASSERT_OK(GenerateAndAddExternalFile( | |
320 | options, {2000, 3002}, ValueType::kTypeValue, file_id++, &true_data)); | |
321 | // A global seqno will be assigned anyway because of the snapshot | |
322 | ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 4); | |
323 | ||
324 | ASSERT_OK(GenerateAndAddExternalFile(options, {1, 20, 40, 100, 150}, | |
325 | ValueType::kTypeValue, file_id++, | |
326 | &true_data)); | |
327 | // A global seqno will be assigned anyway because of the snapshot | |
328 | ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 5); | |
329 | ||
330 | db_->ReleaseSnapshot(snapshot); | |
331 | ||
332 | ASSERT_OK(GenerateAndAddExternalFile( | |
333 | options, {5000, 5001}, ValueType::kTypeValue, file_id++, &true_data)); | |
334 | // No snapshot anymore, no need to assign a seqno | |
335 | ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 5); | |
336 | ||
337 | size_t kcnt = 0; | |
338 | VerifyDBFromMap(true_data, &kcnt, false); | |
339 | } while (ChangeCompactOptions()); | |
340 | } | |
341 | ||
342 | TEST_F(ExternalSSTFileBasicTest, IngestFileWithMultipleValueType) { | |
343 | do { | |
344 | Options options = CurrentOptions(); | |
345 | options.merge_operator.reset(new TestPutOperator()); | |
346 | DestroyAndReopen(options); | |
347 | std::map<std::string, std::string> true_data; | |
348 | ||
349 | int file_id = 1; | |
350 | ||
351 | ASSERT_OK(GenerateAndAddExternalFile(options, {1, 2, 3, 4, 5, 6}, | |
352 | ValueType::kTypeValue, file_id++, | |
353 | &true_data)); | |
354 | // File doesn't overwrite any keys, no seqno needed | |
355 | ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 0); | |
356 | ||
357 | ASSERT_OK(GenerateAndAddExternalFile(options, {10, 11, 12, 13}, | |
358 | ValueType::kTypeValue, file_id++, | |
7c673cae | 359 | &true_data)); |
11fdf7f2 TL |
360 | // File doesn't overwrite any keys, no seqno needed |
361 | ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 0); | |
362 | ||
363 | ASSERT_OK(GenerateAndAddExternalFile( | |
364 | options, {1, 4, 6}, ValueType::kTypeMerge, file_id++, &true_data)); | |
365 | // File overwrites some keys, a seqno will be assigned | |
366 | ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 1); | |
367 | ||
368 | ASSERT_OK(GenerateAndAddExternalFile(options, {11, 15, 19}, | |
369 | ValueType::kTypeDeletion, file_id++, | |
370 | &true_data)); | |
371 | // File overwrites some keys, a seqno will be assigned | |
372 | ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 2); | |
373 | ||
374 | ASSERT_OK(GenerateAndAddExternalFile( | |
375 | options, {120, 130}, ValueType::kTypeMerge, file_id++, &true_data)); | |
376 | // File doesn't overwrite any keys, no seqno needed | |
377 | ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 2); | |
378 | ||
379 | ASSERT_OK(GenerateAndAddExternalFile( | |
380 | options, {1, 130}, ValueType::kTypeDeletion, file_id++, &true_data)); | |
381 | // File overwrites some keys, a seqno will be assigned | |
382 | ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 3); | |
383 | ||
384 | ASSERT_OK(GenerateAndAddExternalFile(options, {120}, | |
385 | {ValueType::kTypeValue}, {{120, 135}}, | |
386 | file_id++, &true_data)); | |
387 | // File overwrites some keys, a seqno will be assigned | |
388 | ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 4); | |
389 | ||
390 | ASSERT_OK(GenerateAndAddExternalFile(options, {}, {}, {{110, 120}}, | |
391 | file_id++, &true_data)); | |
392 | // The range deletion ends on a key, but it doesn't actually delete | |
393 | // this key because the largest key in the range is exclusive. Still, | |
394 | // it counts as an overlap so a new seqno will be assigned. | |
395 | ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 5); | |
396 | ||
397 | ASSERT_OK(GenerateAndAddExternalFile(options, {}, {}, {{100, 109}}, | |
398 | file_id++, &true_data)); | |
399 | // File doesn't overwrite any keys, no seqno needed | |
400 | ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 5); | |
401 | ||
402 | // Write some keys through normal write path | |
403 | for (int i = 0; i < 50; i++) { | |
404 | ASSERT_OK(Put(Key(i), "memtable")); | |
405 | true_data[Key(i)] = "memtable"; | |
406 | } | |
407 | SequenceNumber last_seqno = dbfull()->GetLatestSequenceNumber(); | |
408 | ||
409 | ASSERT_OK(GenerateAndAddExternalFile( | |
410 | options, {60, 61, 62}, ValueType::kTypeValue, file_id++, &true_data)); | |
411 | // File doesn't overwrite any keys, no seqno needed | |
412 | ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno); | |
413 | ||
414 | ASSERT_OK(GenerateAndAddExternalFile( | |
415 | options, {40, 41, 42}, ValueType::kTypeMerge, file_id++, &true_data)); | |
416 | // File overwrites some keys, a seqno will be assigned | |
7c673cae FG |
417 | ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 1); |
418 | ||
11fdf7f2 TL |
419 | ASSERT_OK(GenerateAndAddExternalFile(options, {20, 30, 40}, |
420 | ValueType::kTypeDeletion, file_id++, | |
7c673cae | 421 | &true_data)); |
11fdf7f2 | 422 | // File overwrites some keys, a seqno will be assigned |
7c673cae FG |
423 | ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 2); |
424 | ||
425 | const Snapshot* snapshot = db_->GetSnapshot(); | |
426 | ||
427 | // We will need a seqno for the file regardless if the file overwrite | |
428 | // keys in the DB or not because we have a snapshot | |
11fdf7f2 TL |
429 | ASSERT_OK(GenerateAndAddExternalFile( |
430 | options, {1000, 1002}, ValueType::kTypeMerge, file_id++, &true_data)); | |
7c673cae FG |
431 | // A global seqno will be assigned anyway because of the snapshot |
432 | ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 3); | |
433 | ||
11fdf7f2 TL |
434 | ASSERT_OK(GenerateAndAddExternalFile( |
435 | options, {2000, 3002}, ValueType::kTypeMerge, file_id++, &true_data)); | |
7c673cae FG |
436 | // A global seqno will be assigned anyway because of the snapshot |
437 | ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 4); | |
438 | ||
439 | ASSERT_OK(GenerateAndAddExternalFile(options, {1, 20, 40, 100, 150}, | |
11fdf7f2 TL |
440 | ValueType::kTypeMerge, file_id++, |
441 | &true_data)); | |
7c673cae FG |
442 | // A global seqno will be assigned anyway because of the snapshot |
443 | ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 5); | |
444 | ||
445 | db_->ReleaseSnapshot(snapshot); | |
446 | ||
11fdf7f2 TL |
447 | ASSERT_OK(GenerateAndAddExternalFile( |
448 | options, {5000, 5001}, ValueType::kTypeValue, file_id++, &true_data)); | |
449 | // No snapshot anymore, no need to assign a seqno | |
450 | ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 5); | |
451 | ||
452 | size_t kcnt = 0; | |
453 | VerifyDBFromMap(true_data, &kcnt, false); | |
454 | } while (ChangeCompactOptions()); | |
455 | } | |
456 | ||
457 | TEST_F(ExternalSSTFileBasicTest, IngestFileWithMixedValueType) { | |
458 | do { | |
459 | Options options = CurrentOptions(); | |
460 | options.merge_operator.reset(new TestPutOperator()); | |
461 | DestroyAndReopen(options); | |
462 | std::map<std::string, std::string> true_data; | |
463 | ||
464 | int file_id = 1; | |
465 | ||
466 | ASSERT_OK(GenerateAndAddExternalFile( | |
467 | options, {1, 2, 3, 4, 5, 6}, | |
468 | {ValueType::kTypeValue, ValueType::kTypeMerge, ValueType::kTypeValue, | |
469 | ValueType::kTypeMerge, ValueType::kTypeValue, ValueType::kTypeMerge}, | |
470 | file_id++, &true_data)); | |
471 | // File doesn't overwrite any keys, no seqno needed | |
472 | ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 0); | |
473 | ||
474 | ASSERT_OK(GenerateAndAddExternalFile( | |
475 | options, {10, 11, 12, 13}, | |
476 | {ValueType::kTypeValue, ValueType::kTypeMerge, ValueType::kTypeValue, | |
477 | ValueType::kTypeMerge}, | |
478 | file_id++, &true_data)); | |
479 | // File doesn't overwrite any keys, no seqno needed | |
480 | ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 0); | |
481 | ||
482 | ASSERT_OK(GenerateAndAddExternalFile( | |
483 | options, {1, 4, 6}, {ValueType::kTypeDeletion, ValueType::kTypeValue, | |
484 | ValueType::kTypeMerge}, | |
485 | file_id++, &true_data)); | |
486 | // File overwrites some keys, a seqno will be assigned | |
487 | ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 1); | |
488 | ||
489 | ASSERT_OK(GenerateAndAddExternalFile( | |
490 | options, {11, 15, 19}, {ValueType::kTypeDeletion, ValueType::kTypeMerge, | |
491 | ValueType::kTypeValue}, | |
492 | file_id++, &true_data)); | |
493 | // File overwrites some keys, a seqno will be assigned | |
494 | ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 2); | |
495 | ||
496 | ASSERT_OK(GenerateAndAddExternalFile( | |
497 | options, {120, 130}, {ValueType::kTypeValue, ValueType::kTypeMerge}, | |
498 | file_id++, &true_data)); | |
499 | // File doesn't overwrite any keys, no seqno needed | |
500 | ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 2); | |
501 | ||
502 | ASSERT_OK(GenerateAndAddExternalFile( | |
503 | options, {1, 130}, {ValueType::kTypeMerge, ValueType::kTypeDeletion}, | |
504 | file_id++, &true_data)); | |
505 | // File overwrites some keys, a seqno will be assigned | |
506 | ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 3); | |
507 | ||
508 | ASSERT_OK(GenerateAndAddExternalFile( | |
509 | options, {150, 151, 152}, | |
510 | {ValueType::kTypeValue, ValueType::kTypeMerge, | |
511 | ValueType::kTypeDeletion}, | |
512 | {{150, 160}, {180, 190}}, file_id++, &true_data)); | |
513 | // File doesn't overwrite any keys, no seqno needed | |
514 | ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 3); | |
515 | ||
516 | ASSERT_OK(GenerateAndAddExternalFile( | |
517 | options, {150, 151, 152}, | |
518 | {ValueType::kTypeValue, ValueType::kTypeMerge, ValueType::kTypeValue}, | |
519 | {{200, 250}}, file_id++, &true_data)); | |
520 | // File overwrites some keys, a seqno will be assigned | |
521 | ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 4); | |
522 | ||
523 | ASSERT_OK(GenerateAndAddExternalFile( | |
524 | options, {300, 301, 302}, | |
525 | {ValueType::kTypeValue, ValueType::kTypeMerge, | |
526 | ValueType::kTypeDeletion}, | |
527 | {{1, 2}, {152, 154}}, file_id++, &true_data)); | |
528 | // File overwrites some keys, a seqno will be assigned | |
529 | ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 5); | |
530 | ||
531 | // Write some keys through normal write path | |
532 | for (int i = 0; i < 50; i++) { | |
533 | ASSERT_OK(Put(Key(i), "memtable")); | |
534 | true_data[Key(i)] = "memtable"; | |
535 | } | |
536 | SequenceNumber last_seqno = dbfull()->GetLatestSequenceNumber(); | |
537 | ||
538 | ASSERT_OK(GenerateAndAddExternalFile( | |
539 | options, {60, 61, 62}, | |
540 | {ValueType::kTypeValue, ValueType::kTypeMerge, ValueType::kTypeValue}, | |
541 | file_id++, &true_data)); | |
542 | // File doesn't overwrite any keys, no seqno needed | |
543 | ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno); | |
544 | ||
545 | ASSERT_OK(GenerateAndAddExternalFile( | |
546 | options, {40, 41, 42}, | |
547 | {ValueType::kTypeValue, ValueType::kTypeDeletion, | |
548 | ValueType::kTypeDeletion}, | |
549 | file_id++, &true_data)); | |
550 | // File overwrites some keys, a seqno will be assigned | |
551 | ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 1); | |
552 | ||
553 | ASSERT_OK(GenerateAndAddExternalFile( | |
554 | options, {20, 30, 40}, | |
555 | {ValueType::kTypeDeletion, ValueType::kTypeDeletion, | |
556 | ValueType::kTypeDeletion}, | |
557 | file_id++, &true_data)); | |
558 | // File overwrites some keys, a seqno will be assigned | |
559 | ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 2); | |
560 | ||
561 | const Snapshot* snapshot = db_->GetSnapshot(); | |
562 | ||
563 | // We will need a seqno for the file regardless if the file overwrite | |
564 | // keys in the DB or not because we have a snapshot | |
565 | ASSERT_OK(GenerateAndAddExternalFile( | |
566 | options, {1000, 1002}, {ValueType::kTypeValue, ValueType::kTypeMerge}, | |
567 | file_id++, &true_data)); | |
568 | // A global seqno will be assigned anyway because of the snapshot | |
569 | ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 3); | |
570 | ||
571 | ASSERT_OK(GenerateAndAddExternalFile( | |
572 | options, {2000, 3002}, {ValueType::kTypeValue, ValueType::kTypeMerge}, | |
573 | file_id++, &true_data)); | |
574 | // A global seqno will be assigned anyway because of the snapshot | |
575 | ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 4); | |
576 | ||
577 | ASSERT_OK(GenerateAndAddExternalFile( | |
578 | options, {1, 20, 40, 100, 150}, | |
579 | {ValueType::kTypeDeletion, ValueType::kTypeDeletion, | |
580 | ValueType::kTypeValue, ValueType::kTypeMerge, ValueType::kTypeMerge}, | |
581 | file_id++, &true_data)); | |
582 | // A global seqno will be assigned anyway because of the snapshot | |
583 | ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 5); | |
584 | ||
585 | db_->ReleaseSnapshot(snapshot); | |
586 | ||
587 | ASSERT_OK(GenerateAndAddExternalFile( | |
588 | options, {5000, 5001}, {ValueType::kTypeValue, ValueType::kTypeMerge}, | |
589 | file_id++, &true_data)); | |
7c673cae FG |
590 | // No snapshot anymore, no need to assign a seqno |
591 | ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno + 5); | |
592 | ||
593 | size_t kcnt = 0; | |
594 | VerifyDBFromMap(true_data, &kcnt, false); | |
595 | } while (ChangeCompactOptions()); | |
596 | } | |
597 | ||
598 | TEST_F(ExternalSSTFileBasicTest, FadviseTrigger) { | |
599 | Options options = CurrentOptions(); | |
600 | const int kNumKeys = 10000; | |
601 | ||
602 | size_t total_fadvised_bytes = 0; | |
603 | rocksdb::SyncPoint::GetInstance()->SetCallBack( | |
11fdf7f2 | 604 | "SstFileWriter::Rep::InvalidatePageCache", [&](void* arg) { |
7c673cae FG |
605 | size_t fadvise_size = *(reinterpret_cast<size_t*>(arg)); |
606 | total_fadvised_bytes += fadvise_size; | |
607 | }); | |
608 | rocksdb::SyncPoint::GetInstance()->EnableProcessing(); | |
609 | ||
610 | std::unique_ptr<SstFileWriter> sst_file_writer; | |
611 | ||
612 | std::string sst_file_path = sst_files_dir_ + "file_fadvise_disable.sst"; | |
613 | sst_file_writer.reset( | |
614 | new SstFileWriter(EnvOptions(), options, nullptr, false)); | |
615 | ASSERT_OK(sst_file_writer->Open(sst_file_path)); | |
616 | for (int i = 0; i < kNumKeys; i++) { | |
11fdf7f2 | 617 | ASSERT_OK(sst_file_writer->Put(Key(i), Key(i))); |
7c673cae FG |
618 | } |
619 | ASSERT_OK(sst_file_writer->Finish()); | |
620 | // fadvise disabled | |
621 | ASSERT_EQ(total_fadvised_bytes, 0); | |
622 | ||
7c673cae FG |
623 | sst_file_path = sst_files_dir_ + "file_fadvise_enable.sst"; |
624 | sst_file_writer.reset( | |
625 | new SstFileWriter(EnvOptions(), options, nullptr, true)); | |
626 | ASSERT_OK(sst_file_writer->Open(sst_file_path)); | |
627 | for (int i = 0; i < kNumKeys; i++) { | |
11fdf7f2 | 628 | ASSERT_OK(sst_file_writer->Put(Key(i), Key(i))); |
7c673cae FG |
629 | } |
630 | ASSERT_OK(sst_file_writer->Finish()); | |
631 | // fadvise enabled | |
632 | ASSERT_EQ(total_fadvised_bytes, sst_file_writer->FileSize()); | |
633 | ASSERT_GT(total_fadvised_bytes, 0); | |
634 | ||
635 | rocksdb::SyncPoint::GetInstance()->DisableProcessing(); | |
636 | } | |
637 | ||
11fdf7f2 TL |
638 | TEST_F(ExternalSSTFileBasicTest, IngestionWithRangeDeletions) { |
639 | int kNumLevels = 7; | |
640 | Options options = CurrentOptions(); | |
641 | options.disable_auto_compactions = true; | |
642 | options.num_levels = kNumLevels; | |
643 | Reopen(options); | |
644 | ||
645 | std::map<std::string, std::string> true_data; | |
646 | int file_id = 1; | |
647 | // prevent range deletions from being dropped due to becoming obsolete. | |
648 | const Snapshot* snapshot = db_->GetSnapshot(); | |
649 | ||
650 | // range del [0, 50) in L6 file, [50, 100) in L0 file, [100, 150) in memtable | |
651 | for (int i = 0; i < 3; i++) { | |
652 | if (i != 0) { | |
653 | db_->Flush(FlushOptions()); | |
654 | if (i == 1) { | |
655 | MoveFilesToLevel(kNumLevels - 1); | |
656 | } | |
657 | } | |
658 | ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), | |
659 | Key(50 * i), Key(50 * (i + 1)))); | |
660 | } | |
661 | ASSERT_EQ(1, NumTableFilesAtLevel(0)); | |
662 | ASSERT_EQ(0, NumTableFilesAtLevel(kNumLevels - 2)); | |
663 | ASSERT_EQ(1, NumTableFilesAtLevel(kNumLevels - 1)); | |
664 | ||
665 | // overlaps with L0 file but not memtable, so flush is skipped and file is | |
666 | // ingested into L0 | |
667 | SequenceNumber last_seqno = dbfull()->GetLatestSequenceNumber(); | |
668 | ASSERT_OK(GenerateAndAddExternalFile( | |
669 | options, {60, 90}, {ValueType::kTypeValue, ValueType::kTypeValue}, | |
670 | {{65, 70}, {70, 85}}, file_id++, &true_data)); | |
671 | ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), ++last_seqno); | |
672 | ASSERT_EQ(2, NumTableFilesAtLevel(0)); | |
673 | ASSERT_EQ(0, NumTableFilesAtLevel(kNumLevels - 2)); | |
674 | ASSERT_EQ(1, NumTableFilesAtLevel(options.num_levels - 1)); | |
675 | ||
676 | // overlaps with L6 file but not memtable or L0 file, so flush is skipped and | |
677 | // file is ingested into L5 | |
678 | ASSERT_OK(GenerateAndAddExternalFile( | |
679 | options, {10, 40}, {ValueType::kTypeValue, ValueType::kTypeValue}, | |
680 | file_id++, &true_data)); | |
681 | ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), ++last_seqno); | |
682 | ASSERT_EQ(2, NumTableFilesAtLevel(0)); | |
683 | ASSERT_EQ(1, NumTableFilesAtLevel(kNumLevels - 2)); | |
684 | ASSERT_EQ(1, NumTableFilesAtLevel(options.num_levels - 1)); | |
685 | ||
686 | // overlaps with L5 file but not memtable or L0 file, so flush is skipped and | |
687 | // file is ingested into L4 | |
688 | ASSERT_OK(GenerateAndAddExternalFile(options, {}, {}, {{5, 15}}, file_id++, | |
689 | &true_data)); | |
690 | ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), ++last_seqno); | |
691 | ASSERT_EQ(2, NumTableFilesAtLevel(0)); | |
692 | ASSERT_EQ(1, NumTableFilesAtLevel(kNumLevels - 2)); | |
693 | ASSERT_EQ(1, NumTableFilesAtLevel(options.num_levels - 2)); | |
694 | ASSERT_EQ(1, NumTableFilesAtLevel(options.num_levels - 1)); | |
695 | ||
696 | // ingested file overlaps with memtable, so flush is triggered before the file | |
697 | // is ingested such that the ingested data is considered newest. So L0 file | |
698 | // count increases by two. | |
699 | ASSERT_OK(GenerateAndAddExternalFile( | |
700 | options, {100, 140}, {ValueType::kTypeValue, ValueType::kTypeValue}, | |
701 | file_id++, &true_data)); | |
702 | ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), ++last_seqno); | |
703 | ASSERT_EQ(4, NumTableFilesAtLevel(0)); | |
704 | ASSERT_EQ(1, NumTableFilesAtLevel(kNumLevels - 2)); | |
705 | ASSERT_EQ(1, NumTableFilesAtLevel(options.num_levels - 1)); | |
706 | ||
707 | // snapshot unneeded now that all range deletions are persisted | |
708 | db_->ReleaseSnapshot(snapshot); | |
709 | ||
710 | // overlaps with nothing, so places at bottom level and skips incrementing | |
711 | // seqnum. | |
712 | ASSERT_OK(GenerateAndAddExternalFile( | |
713 | options, {151, 175}, {ValueType::kTypeValue, ValueType::kTypeValue}, | |
714 | {{160, 200}}, file_id++, &true_data)); | |
715 | ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), last_seqno); | |
716 | ASSERT_EQ(4, NumTableFilesAtLevel(0)); | |
717 | ASSERT_EQ(1, NumTableFilesAtLevel(kNumLevels - 2)); | |
718 | ASSERT_EQ(2, NumTableFilesAtLevel(options.num_levels - 1)); | |
719 | } | |
720 | ||
7c673cae FG |
721 | #endif // ROCKSDB_LITE |
722 | ||
723 | } // namespace rocksdb | |
724 | ||
725 | int main(int argc, char** argv) { | |
726 | rocksdb::port::InstallStackTraceHandler(); | |
727 | ::testing::InitGoogleTest(&argc, argv); | |
728 | return RUN_ALL_TESTS(); | |
729 | } |