]>
Commit | Line | Data |
---|---|---|
7c673cae | 1 | // Copyright (c) 2016-present, Facebook, Inc. All rights reserved. |
11fdf7f2 TL |
2 | // This source code is licensed under both the GPLv2 (found in the |
3 | // COPYING file in the root directory) and Apache 2.0 License | |
4 | // (found in the LICENSE.Apache file in the root directory). | |
7c673cae FG |
5 | |
6 | #ifndef ROCKSDB_LITE | |
7 | ||
8 | #include <algorithm> | |
9 | #include <string> | |
10 | #include <vector> | |
11 | ||
12 | #include "db/db_impl.h" | |
13 | #include "db/db_test_util.h" | |
14 | #include "rocksdb/comparator.h" | |
15 | #include "rocksdb/db.h" | |
16 | #include "rocksdb/transaction_log.h" | |
17 | #include "util/file_util.h" | |
18 | #include "util/string_util.h" | |
19 | ||
20 | namespace rocksdb { | |
21 | ||
22 | #ifndef ROCKSDB_LITE | |
23 | class RepairTest : public DBTestBase { | |
24 | public: | |
25 | RepairTest() : DBTestBase("/repair_test") {} | |
26 | ||
27 | std::string GetFirstSstPath() { | |
28 | uint64_t manifest_size; | |
29 | std::vector<std::string> files; | |
30 | db_->GetLiveFiles(files, &manifest_size); | |
31 | auto sst_iter = | |
32 | std::find_if(files.begin(), files.end(), [](const std::string& file) { | |
33 | uint64_t number; | |
34 | FileType type; | |
35 | bool ok = ParseFileName(file, &number, &type); | |
36 | return ok && type == kTableFile; | |
37 | }); | |
38 | return sst_iter == files.end() ? "" : dbname_ + *sst_iter; | |
39 | } | |
40 | }; | |
41 | ||
42 | TEST_F(RepairTest, LostManifest) { | |
43 | // Add a couple SST files, delete the manifest, and verify RepairDB() saves | |
44 | // the day. | |
45 | Put("key", "val"); | |
46 | Flush(); | |
47 | Put("key2", "val2"); | |
48 | Flush(); | |
49 | // Need to get path before Close() deletes db_, but delete it after Close() to | |
50 | // ensure Close() didn't change the manifest. | |
51 | std::string manifest_path = | |
52 | DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo()); | |
53 | ||
54 | Close(); | |
55 | ASSERT_OK(env_->FileExists(manifest_path)); | |
56 | ASSERT_OK(env_->DeleteFile(manifest_path)); | |
57 | ASSERT_OK(RepairDB(dbname_, CurrentOptions())); | |
58 | Reopen(CurrentOptions()); | |
59 | ||
60 | ASSERT_EQ(Get("key"), "val"); | |
61 | ASSERT_EQ(Get("key2"), "val2"); | |
62 | } | |
63 | ||
64 | TEST_F(RepairTest, CorruptManifest) { | |
65 | // Manifest is in an invalid format. Expect a full recovery. | |
66 | Put("key", "val"); | |
67 | Flush(); | |
68 | Put("key2", "val2"); | |
69 | Flush(); | |
70 | // Need to get path before Close() deletes db_, but overwrite it after Close() | |
71 | // to ensure Close() didn't change the manifest. | |
72 | std::string manifest_path = | |
73 | DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo()); | |
74 | ||
75 | Close(); | |
76 | ASSERT_OK(env_->FileExists(manifest_path)); | |
11fdf7f2 | 77 | CreateFile(env_, manifest_path, "blah", false /* use_fsync */); |
7c673cae FG |
78 | ASSERT_OK(RepairDB(dbname_, CurrentOptions())); |
79 | Reopen(CurrentOptions()); | |
80 | ||
81 | ASSERT_EQ(Get("key"), "val"); | |
82 | ASSERT_EQ(Get("key2"), "val2"); | |
83 | } | |
84 | ||
85 | TEST_F(RepairTest, IncompleteManifest) { | |
86 | // In this case, the manifest is valid but does not reference all of the SST | |
87 | // files. Expect a full recovery. | |
88 | Put("key", "val"); | |
89 | Flush(); | |
90 | std::string orig_manifest_path = | |
91 | DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo()); | |
92 | CopyFile(orig_manifest_path, orig_manifest_path + ".tmp"); | |
93 | Put("key2", "val2"); | |
94 | Flush(); | |
95 | // Need to get path before Close() deletes db_, but overwrite it after Close() | |
96 | // to ensure Close() didn't change the manifest. | |
97 | std::string new_manifest_path = | |
98 | DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo()); | |
99 | ||
100 | Close(); | |
101 | ASSERT_OK(env_->FileExists(new_manifest_path)); | |
102 | // Replace the manifest with one that is only aware of the first SST file. | |
103 | CopyFile(orig_manifest_path + ".tmp", new_manifest_path); | |
104 | ASSERT_OK(RepairDB(dbname_, CurrentOptions())); | |
105 | Reopen(CurrentOptions()); | |
106 | ||
107 | ASSERT_EQ(Get("key"), "val"); | |
108 | ASSERT_EQ(Get("key2"), "val2"); | |
109 | } | |
110 | ||
11fdf7f2 TL |
111 | TEST_F(RepairTest, PostRepairSstFileNumbering) { |
112 | // Verify after a DB is repaired, new files will be assigned higher numbers | |
113 | // than old files. | |
114 | Put("key", "val"); | |
115 | Flush(); | |
116 | Put("key2", "val2"); | |
117 | Flush(); | |
118 | uint64_t pre_repair_file_num = dbfull()->TEST_Current_Next_FileNo(); | |
119 | Close(); | |
120 | ||
121 | ASSERT_OK(RepairDB(dbname_, CurrentOptions())); | |
122 | ||
123 | Reopen(CurrentOptions()); | |
124 | uint64_t post_repair_file_num = dbfull()->TEST_Current_Next_FileNo(); | |
125 | ASSERT_GE(post_repair_file_num, pre_repair_file_num); | |
126 | } | |
127 | ||
7c673cae FG |
128 | TEST_F(RepairTest, LostSst) { |
129 | // Delete one of the SST files but preserve the manifest that refers to it, | |
130 | // then verify the DB is still usable for the intact SST. | |
131 | Put("key", "val"); | |
132 | Flush(); | |
133 | Put("key2", "val2"); | |
134 | Flush(); | |
135 | auto sst_path = GetFirstSstPath(); | |
136 | ASSERT_FALSE(sst_path.empty()); | |
137 | ASSERT_OK(env_->DeleteFile(sst_path)); | |
138 | ||
139 | Close(); | |
140 | ASSERT_OK(RepairDB(dbname_, CurrentOptions())); | |
141 | Reopen(CurrentOptions()); | |
142 | ||
143 | // Exactly one of the key-value pairs should be in the DB now. | |
144 | ASSERT_TRUE((Get("key") == "val") != (Get("key2") == "val2")); | |
145 | } | |
146 | ||
147 | TEST_F(RepairTest, CorruptSst) { | |
148 | // Corrupt one of the SST files but preserve the manifest that refers to it, | |
149 | // then verify the DB is still usable for the intact SST. | |
150 | Put("key", "val"); | |
151 | Flush(); | |
152 | Put("key2", "val2"); | |
153 | Flush(); | |
154 | auto sst_path = GetFirstSstPath(); | |
155 | ASSERT_FALSE(sst_path.empty()); | |
11fdf7f2 | 156 | CreateFile(env_, sst_path, "blah", false /* use_fsync */); |
7c673cae FG |
157 | |
158 | Close(); | |
159 | ASSERT_OK(RepairDB(dbname_, CurrentOptions())); | |
160 | Reopen(CurrentOptions()); | |
161 | ||
162 | // Exactly one of the key-value pairs should be in the DB now. | |
163 | ASSERT_TRUE((Get("key") == "val") != (Get("key2") == "val2")); | |
164 | } | |
165 | ||
166 | TEST_F(RepairTest, UnflushedSst) { | |
167 | // This test case invokes repair while some data is unflushed, then verifies | |
168 | // that data is in the db. | |
169 | Put("key", "val"); | |
170 | VectorLogPtr wal_files; | |
171 | ASSERT_OK(dbfull()->GetSortedWalFiles(wal_files)); | |
172 | ASSERT_EQ(wal_files.size(), 1); | |
173 | uint64_t total_ssts_size; | |
174 | GetAllSSTFiles(&total_ssts_size); | |
175 | ASSERT_EQ(total_ssts_size, 0); | |
176 | // Need to get path before Close() deletes db_, but delete it after Close() to | |
177 | // ensure Close() didn't change the manifest. | |
178 | std::string manifest_path = | |
179 | DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo()); | |
180 | ||
181 | Close(); | |
182 | ASSERT_OK(env_->FileExists(manifest_path)); | |
183 | ASSERT_OK(env_->DeleteFile(manifest_path)); | |
184 | ASSERT_OK(RepairDB(dbname_, CurrentOptions())); | |
185 | Reopen(CurrentOptions()); | |
186 | ||
187 | ASSERT_OK(dbfull()->GetSortedWalFiles(wal_files)); | |
188 | ASSERT_EQ(wal_files.size(), 0); | |
189 | GetAllSSTFiles(&total_ssts_size); | |
190 | ASSERT_GT(total_ssts_size, 0); | |
191 | ASSERT_EQ(Get("key"), "val"); | |
192 | } | |
193 | ||
11fdf7f2 TL |
194 | TEST_F(RepairTest, SeparateWalDir) { |
195 | do { | |
196 | Options options = CurrentOptions(); | |
197 | DestroyAndReopen(options); | |
198 | Put("key", "val"); | |
199 | Put("foo", "bar"); | |
200 | VectorLogPtr wal_files; | |
201 | ASSERT_OK(dbfull()->GetSortedWalFiles(wal_files)); | |
202 | ASSERT_EQ(wal_files.size(), 1); | |
203 | uint64_t total_ssts_size; | |
204 | GetAllSSTFiles(&total_ssts_size); | |
205 | ASSERT_EQ(total_ssts_size, 0); | |
206 | std::string manifest_path = | |
207 | DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo()); | |
208 | ||
209 | Close(); | |
210 | ASSERT_OK(env_->FileExists(manifest_path)); | |
211 | ASSERT_OK(env_->DeleteFile(manifest_path)); | |
212 | ASSERT_OK(RepairDB(dbname_, options)); | |
213 | ||
214 | // make sure that all WALs are converted to SSTables. | |
215 | options.wal_dir = ""; | |
216 | ||
217 | Reopen(options); | |
218 | ASSERT_OK(dbfull()->GetSortedWalFiles(wal_files)); | |
219 | ASSERT_EQ(wal_files.size(), 0); | |
220 | GetAllSSTFiles(&total_ssts_size); | |
221 | ASSERT_GT(total_ssts_size, 0); | |
222 | ASSERT_EQ(Get("key"), "val"); | |
223 | ASSERT_EQ(Get("foo"), "bar"); | |
224 | ||
225 | } while(ChangeWalOptions()); | |
226 | } | |
227 | ||
7c673cae FG |
228 | TEST_F(RepairTest, RepairMultipleColumnFamilies) { |
229 | // Verify repair logic associates SST files with their original column | |
230 | // families. | |
231 | const int kNumCfs = 3; | |
232 | const int kEntriesPerCf = 2; | |
233 | DestroyAndReopen(CurrentOptions()); | |
234 | CreateAndReopenWithCF({"pikachu1", "pikachu2"}, CurrentOptions()); | |
235 | for (int i = 0; i < kNumCfs; ++i) { | |
236 | for (int j = 0; j < kEntriesPerCf; ++j) { | |
237 | Put(i, "key" + ToString(j), "val" + ToString(j)); | |
238 | if (j == kEntriesPerCf - 1 && i == kNumCfs - 1) { | |
239 | // Leave one unflushed so we can verify WAL entries are properly | |
240 | // associated with column families. | |
241 | continue; | |
242 | } | |
243 | Flush(i); | |
244 | } | |
245 | } | |
246 | ||
247 | // Need to get path before Close() deletes db_, but delete it after Close() to | |
248 | // ensure Close() doesn't re-create the manifest. | |
249 | std::string manifest_path = | |
250 | DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo()); | |
251 | Close(); | |
252 | ASSERT_OK(env_->FileExists(manifest_path)); | |
253 | ASSERT_OK(env_->DeleteFile(manifest_path)); | |
254 | ||
255 | ASSERT_OK(RepairDB(dbname_, CurrentOptions())); | |
256 | ||
257 | ReopenWithColumnFamilies({"default", "pikachu1", "pikachu2"}, | |
258 | CurrentOptions()); | |
259 | for (int i = 0; i < kNumCfs; ++i) { | |
260 | for (int j = 0; j < kEntriesPerCf; ++j) { | |
261 | ASSERT_EQ(Get(i, "key" + ToString(j)), "val" + ToString(j)); | |
262 | } | |
263 | } | |
264 | } | |
265 | ||
266 | TEST_F(RepairTest, RepairColumnFamilyOptions) { | |
267 | // Verify repair logic uses correct ColumnFamilyOptions when repairing a | |
268 | // database with different options for column families. | |
269 | const int kNumCfs = 2; | |
270 | const int kEntriesPerCf = 2; | |
271 | ||
272 | Options opts(CurrentOptions()), rev_opts(CurrentOptions()); | |
273 | opts.comparator = BytewiseComparator(); | |
274 | rev_opts.comparator = ReverseBytewiseComparator(); | |
275 | ||
276 | DestroyAndReopen(opts); | |
277 | CreateColumnFamilies({"reverse"}, rev_opts); | |
278 | ReopenWithColumnFamilies({"default", "reverse"}, | |
279 | std::vector<Options>{opts, rev_opts}); | |
280 | for (int i = 0; i < kNumCfs; ++i) { | |
281 | for (int j = 0; j < kEntriesPerCf; ++j) { | |
282 | Put(i, "key" + ToString(j), "val" + ToString(j)); | |
283 | if (i == kNumCfs - 1 && j == kEntriesPerCf - 1) { | |
284 | // Leave one unflushed so we can verify RepairDB's flush logic | |
285 | continue; | |
286 | } | |
287 | Flush(i); | |
288 | } | |
289 | } | |
290 | Close(); | |
291 | ||
292 | // RepairDB() records the comparator in the manifest, and DB::Open would fail | |
293 | // if a different comparator were used. | |
294 | ASSERT_OK(RepairDB(dbname_, opts, {{"default", opts}, {"reverse", rev_opts}}, | |
295 | opts /* unknown_cf_opts */)); | |
296 | ASSERT_OK(TryReopenWithColumnFamilies({"default", "reverse"}, | |
297 | std::vector<Options>{opts, rev_opts})); | |
298 | for (int i = 0; i < kNumCfs; ++i) { | |
299 | for (int j = 0; j < kEntriesPerCf; ++j) { | |
300 | ASSERT_EQ(Get(i, "key" + ToString(j)), "val" + ToString(j)); | |
301 | } | |
302 | } | |
303 | ||
304 | // Examine table properties to verify RepairDB() used the right options when | |
305 | // converting WAL->SST | |
306 | TablePropertiesCollection fname_to_props; | |
307 | db_->GetPropertiesOfAllTables(handles_[1], &fname_to_props); | |
308 | ASSERT_EQ(fname_to_props.size(), 2U); | |
309 | for (const auto& fname_and_props : fname_to_props) { | |
310 | std::string comparator_name ( | |
311 | InternalKeyComparator(rev_opts.comparator).Name()); | |
312 | comparator_name = comparator_name.substr(comparator_name.find(':') + 1); | |
313 | ASSERT_EQ(comparator_name, | |
314 | fname_and_props.second->comparator_name); | |
315 | } | |
494da23a | 316 | Close(); |
7c673cae FG |
317 | |
318 | // Also check comparator when it's provided via "unknown" CF options | |
319 | ASSERT_OK(RepairDB(dbname_, opts, {{"default", opts}}, | |
320 | rev_opts /* unknown_cf_opts */)); | |
321 | ASSERT_OK(TryReopenWithColumnFamilies({"default", "reverse"}, | |
322 | std::vector<Options>{opts, rev_opts})); | |
323 | for (int i = 0; i < kNumCfs; ++i) { | |
324 | for (int j = 0; j < kEntriesPerCf; ++j) { | |
325 | ASSERT_EQ(Get(i, "key" + ToString(j)), "val" + ToString(j)); | |
326 | } | |
327 | } | |
328 | } | |
329 | ||
11fdf7f2 TL |
330 | TEST_F(RepairTest, DbNameContainsTrailingSlash) { |
331 | { | |
332 | bool tmp; | |
333 | if (env_->AreFilesSame("", "", &tmp).IsNotSupported()) { | |
334 | fprintf(stderr, | |
335 | "skipping RepairTest.DbNameContainsTrailingSlash due to " | |
336 | "unsupported Env::AreFilesSame\n"); | |
337 | return; | |
338 | } | |
339 | } | |
340 | ||
341 | Put("key", "val"); | |
342 | Flush(); | |
343 | Close(); | |
344 | ||
345 | ASSERT_OK(RepairDB(dbname_ + "/", CurrentOptions())); | |
346 | Reopen(CurrentOptions()); | |
347 | ASSERT_EQ(Get("key"), "val"); | |
348 | } | |
7c673cae FG |
349 | #endif // ROCKSDB_LITE |
350 | } // namespace rocksdb | |
351 | ||
352 | int main(int argc, char** argv) { | |
353 | ::testing::InitGoogleTest(&argc, argv); | |
354 | return RUN_ALL_TESTS(); | |
355 | } | |
356 | ||
357 | #else | |
358 | #include <stdio.h> | |
359 | ||
11fdf7f2 | 360 | int main(int /*argc*/, char** /*argv*/) { |
7c673cae FG |
361 | fprintf(stderr, "SKIPPED as RepairDB is not supported in ROCKSDB_LITE\n"); |
362 | return 0; | |
363 | } | |
364 | ||
365 | #endif // ROCKSDB_LITE |