]>
Commit | Line | Data |
---|---|---|
7c673cae | 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. |
11fdf7f2 TL |
2 | // This source code is licensed under both the GPLv2 (found in the |
3 | // COPYING file in the root directory) and Apache 2.0 License | |
4 | // (found in the LICENSE.Apache file in the root directory). | |
7c673cae FG |
5 | // |
6 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. | |
7 | // Use of this source code is governed by a BSD-style license that can be | |
8 | // found in the LICENSE file. See the AUTHORS file for names of contributors. | |
9 | ||
10 | #include "db/db_test_util.h" | |
11 | #include "port/stack_trace.h" | |
20effc67 | 12 | #include "util/random.h" |
7c673cae | 13 | |
f67539c2 | 14 | namespace ROCKSDB_NAMESPACE { |
7c673cae FG |
15 | |
16 | class DBIOFailureTest : public DBTestBase { | |
17 | public: | |
20effc67 TL |
18 | DBIOFailureTest() |
19 | : DBTestBase("/db_io_failure_test", /*env_do_fsync=*/true) {} | |
7c673cae FG |
20 | }; |
21 | ||
22 | #ifndef ROCKSDB_LITE | |
23 | // Check that number of files does not grow when writes are dropped | |
24 | TEST_F(DBIOFailureTest, DropWrites) { | |
25 | do { | |
26 | Options options = CurrentOptions(); | |
27 | options.env = env_; | |
28 | options.paranoid_checks = false; | |
29 | Reopen(options); | |
30 | ||
31 | ASSERT_OK(Put("foo", "v1")); | |
32 | ASSERT_EQ("v1", Get("foo")); | |
33 | Compact("a", "z"); | |
34 | const size_t num_files = CountFiles(); | |
35 | // Force out-of-space errors | |
36 | env_->drop_writes_.store(true, std::memory_order_release); | |
37 | env_->sleep_counter_.Reset(); | |
20effc67 | 38 | env_->SetMockSleep(); |
7c673cae FG |
39 | for (int i = 0; i < 5; i++) { |
40 | if (option_config_ != kUniversalCompactionMultiLevel && | |
41 | option_config_ != kUniversalSubcompactions) { | |
42 | for (int level = 0; level < dbfull()->NumberLevels(); level++) { | |
43 | if (level > 0 && level == dbfull()->NumberLevels() - 1) { | |
44 | break; | |
45 | } | |
46 | dbfull()->TEST_CompactRange(level, nullptr, nullptr, nullptr, | |
47 | true /* disallow trivial move */); | |
48 | } | |
49 | } else { | |
50 | dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr); | |
51 | } | |
52 | } | |
53 | ||
54 | std::string property_value; | |
55 | ASSERT_TRUE(db_->GetProperty("rocksdb.background-errors", &property_value)); | |
56 | ASSERT_EQ("5", property_value); | |
57 | ||
58 | env_->drop_writes_.store(false, std::memory_order_release); | |
59 | ASSERT_LT(CountFiles(), num_files + 3); | |
60 | ||
61 | // Check that compaction attempts slept after errors | |
62 | // TODO @krad: Figure out why ASSERT_EQ 5 keeps failing in certain compiler | |
63 | // versions | |
64 | ASSERT_GE(env_->sleep_counter_.Read(), 4); | |
65 | } while (ChangeCompactOptions()); | |
66 | } | |
67 | ||
68 | // Check background error counter bumped on flush failures. | |
69 | TEST_F(DBIOFailureTest, DropWritesFlush) { | |
70 | do { | |
71 | Options options = CurrentOptions(); | |
72 | options.env = env_; | |
73 | options.max_background_flushes = 1; | |
74 | Reopen(options); | |
75 | ||
76 | ASSERT_OK(Put("foo", "v1")); | |
77 | // Force out-of-space errors | |
78 | env_->drop_writes_.store(true, std::memory_order_release); | |
79 | ||
80 | std::string property_value; | |
81 | // Background error count is 0 now. | |
82 | ASSERT_TRUE(db_->GetProperty("rocksdb.background-errors", &property_value)); | |
83 | ASSERT_EQ("0", property_value); | |
84 | ||
85 | dbfull()->TEST_FlushMemTable(true); | |
86 | ||
87 | ASSERT_TRUE(db_->GetProperty("rocksdb.background-errors", &property_value)); | |
88 | ASSERT_EQ("1", property_value); | |
89 | ||
90 | env_->drop_writes_.store(false, std::memory_order_release); | |
91 | } while (ChangeCompactOptions()); | |
92 | } | |
7c673cae FG |
93 | |
94 | // Check that CompactRange() returns failure if there is not enough space left | |
95 | // on device | |
96 | TEST_F(DBIOFailureTest, NoSpaceCompactRange) { | |
97 | do { | |
98 | Options options = CurrentOptions(); | |
99 | options.env = env_; | |
100 | options.disable_auto_compactions = true; | |
101 | Reopen(options); | |
102 | ||
103 | // generate 5 tables | |
104 | for (int i = 0; i < 5; ++i) { | |
105 | ASSERT_OK(Put(Key(i), Key(i) + "v")); | |
106 | ASSERT_OK(Flush()); | |
107 | } | |
108 | ||
109 | // Force out-of-space errors | |
110 | env_->no_space_.store(true, std::memory_order_release); | |
111 | ||
112 | Status s = dbfull()->TEST_CompactRange(0, nullptr, nullptr, nullptr, | |
113 | true /* disallow trivial move */); | |
114 | ASSERT_TRUE(s.IsIOError()); | |
115 | ASSERT_TRUE(s.IsNoSpace()); | |
116 | ||
117 | env_->no_space_.store(false, std::memory_order_release); | |
118 | } while (ChangeCompactOptions()); | |
119 | } | |
494da23a | 120 | #endif // ROCKSDB_LITE |
7c673cae FG |
121 | |
122 | TEST_F(DBIOFailureTest, NonWritableFileSystem) { | |
123 | do { | |
124 | Options options = CurrentOptions(); | |
125 | options.write_buffer_size = 4096; | |
126 | options.arena_block_size = 4096; | |
127 | options.env = env_; | |
128 | Reopen(options); | |
129 | ASSERT_OK(Put("foo", "v1")); | |
130 | env_->non_writeable_rate_.store(100); | |
131 | std::string big(100000, 'x'); | |
132 | int errors = 0; | |
133 | for (int i = 0; i < 20; i++) { | |
134 | if (!Put("foo", big).ok()) { | |
135 | errors++; | |
136 | env_->SleepForMicroseconds(100000); | |
137 | } | |
138 | } | |
139 | ASSERT_GT(errors, 0); | |
140 | env_->non_writeable_rate_.store(0); | |
141 | } while (ChangeCompactOptions()); | |
142 | } | |
143 | ||
144 | #ifndef ROCKSDB_LITE | |
145 | TEST_F(DBIOFailureTest, ManifestWriteError) { | |
146 | // Test for the following problem: | |
147 | // (a) Compaction produces file F | |
148 | // (b) Log record containing F is written to MANIFEST file, but Sync() fails | |
149 | // (c) GC deletes F | |
150 | // (d) After reopening DB, reads fail since deleted F is named in log record | |
151 | ||
152 | // We iterate twice. In the second iteration, everything is the | |
153 | // same except the log record never makes it to the MANIFEST file. | |
154 | for (int iter = 0; iter < 2; iter++) { | |
155 | std::atomic<bool>* error_type = (iter == 0) ? &env_->manifest_sync_error_ | |
156 | : &env_->manifest_write_error_; | |
157 | ||
158 | // Insert foo=>bar mapping | |
159 | Options options = CurrentOptions(); | |
160 | options.env = env_; | |
161 | options.create_if_missing = true; | |
162 | options.error_if_exists = false; | |
163 | options.paranoid_checks = true; | |
164 | DestroyAndReopen(options); | |
165 | ASSERT_OK(Put("foo", "bar")); | |
166 | ASSERT_EQ("bar", Get("foo")); | |
167 | ||
168 | // Memtable compaction (will succeed) | |
169 | Flush(); | |
170 | ASSERT_EQ("bar", Get("foo")); | |
171 | const int last = 2; | |
172 | MoveFilesToLevel(2); | |
173 | ASSERT_EQ(NumTableFilesAtLevel(last), 1); // foo=>bar is now in last level | |
174 | ||
175 | // Merging compaction (will fail) | |
176 | error_type->store(true, std::memory_order_release); | |
177 | dbfull()->TEST_CompactRange(last, nullptr, nullptr); // Should fail | |
178 | ASSERT_EQ("bar", Get("foo")); | |
179 | ||
180 | error_type->store(false, std::memory_order_release); | |
181 | ||
182 | // Since paranoid_checks=true, writes should fail | |
183 | ASSERT_NOK(Put("foo2", "bar2")); | |
184 | ||
185 | // Recovery: should not lose data | |
186 | ASSERT_EQ("bar", Get("foo")); | |
187 | ||
188 | // Try again with paranoid_checks=false | |
189 | Close(); | |
190 | options.paranoid_checks = false; | |
191 | Reopen(options); | |
192 | ||
193 | // Merging compaction (will fail) | |
194 | error_type->store(true, std::memory_order_release); | |
195 | dbfull()->TEST_CompactRange(last, nullptr, nullptr); // Should fail | |
196 | ASSERT_EQ("bar", Get("foo")); | |
197 | ||
198 | // Recovery: should not lose data | |
199 | error_type->store(false, std::memory_order_release); | |
200 | Reopen(options); | |
201 | ASSERT_EQ("bar", Get("foo")); | |
202 | ||
203 | // Since paranoid_checks=false, writes should succeed | |
204 | ASSERT_OK(Put("foo2", "bar2")); | |
205 | ASSERT_EQ("bar", Get("foo")); | |
206 | ASSERT_EQ("bar2", Get("foo2")); | |
207 | } | |
208 | } | |
7c673cae FG |
209 | |
210 | TEST_F(DBIOFailureTest, PutFailsParanoid) { | |
211 | // Test the following: | |
212 | // (a) A random put fails in paranoid mode (simulate by sync fail) | |
213 | // (b) All other puts have to fail, even if writes would succeed | |
214 | // (c) All of that should happen ONLY if paranoid_checks = true | |
215 | ||
216 | Options options = CurrentOptions(); | |
217 | options.env = env_; | |
218 | options.create_if_missing = true; | |
219 | options.error_if_exists = false; | |
220 | options.paranoid_checks = true; | |
221 | DestroyAndReopen(options); | |
222 | CreateAndReopenWithCF({"pikachu"}, options); | |
223 | Status s; | |
224 | ||
225 | ASSERT_OK(Put(1, "foo", "bar")); | |
226 | ASSERT_OK(Put(1, "foo1", "bar1")); | |
227 | // simulate error | |
228 | env_->log_write_error_.store(true, std::memory_order_release); | |
229 | s = Put(1, "foo2", "bar2"); | |
230 | ASSERT_TRUE(!s.ok()); | |
231 | env_->log_write_error_.store(false, std::memory_order_release); | |
232 | s = Put(1, "foo3", "bar3"); | |
233 | // the next put should fail, too | |
234 | ASSERT_TRUE(!s.ok()); | |
235 | // but we're still able to read | |
236 | ASSERT_EQ("bar", Get(1, "foo")); | |
237 | ||
238 | // do the same thing with paranoid checks off | |
239 | options.paranoid_checks = false; | |
240 | DestroyAndReopen(options); | |
241 | CreateAndReopenWithCF({"pikachu"}, options); | |
242 | ||
243 | ASSERT_OK(Put(1, "foo", "bar")); | |
244 | ASSERT_OK(Put(1, "foo1", "bar1")); | |
245 | // simulate error | |
246 | env_->log_write_error_.store(true, std::memory_order_release); | |
247 | s = Put(1, "foo2", "bar2"); | |
248 | ASSERT_TRUE(!s.ok()); | |
249 | env_->log_write_error_.store(false, std::memory_order_release); | |
250 | s = Put(1, "foo3", "bar3"); | |
251 | // the next put should NOT fail | |
252 | ASSERT_TRUE(s.ok()); | |
253 | } | |
11fdf7f2 TL |
254 | #if !(defined NDEBUG) || !defined(OS_WIN) |
255 | TEST_F(DBIOFailureTest, FlushSstRangeSyncError) { | |
256 | Options options = CurrentOptions(); | |
257 | options.env = env_; | |
258 | options.create_if_missing = true; | |
259 | options.error_if_exists = false; | |
260 | options.paranoid_checks = true; | |
261 | options.write_buffer_size = 256 * 1024 * 1024; | |
262 | options.writable_file_max_buffer_size = 128 * 1024; | |
263 | options.bytes_per_sync = 128 * 1024; | |
264 | options.level0_file_num_compaction_trigger = 4; | |
265 | options.memtable_factory.reset(new SpecialSkipListFactory(10)); | |
266 | BlockBasedTableOptions table_options; | |
267 | table_options.filter_policy.reset(NewBloomFilterPolicy(10)); | |
268 | options.table_factory.reset(NewBlockBasedTableFactory(table_options)); | |
269 | ||
270 | DestroyAndReopen(options); | |
271 | CreateAndReopenWithCF({"pikachu"}, options); | |
272 | Status s; | |
273 | ||
274 | std::atomic<int> range_sync_called(0); | |
f67539c2 | 275 | ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( |
11fdf7f2 TL |
276 | "SpecialEnv::SStableFile::RangeSync", [&](void* arg) { |
277 | if (range_sync_called.fetch_add(1) == 0) { | |
278 | Status* st = static_cast<Status*>(arg); | |
279 | *st = Status::IOError("range sync dummy error"); | |
280 | } | |
281 | }); | |
f67539c2 | 282 | ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); |
11fdf7f2 TL |
283 | |
284 | Random rnd(301); | |
285 | std::string rnd_str = | |
20effc67 TL |
286 | rnd.RandomString(static_cast<int>(options.bytes_per_sync / 2)); |
287 | std::string rnd_str_512kb = rnd.RandomString(512 * 1024); | |
11fdf7f2 TL |
288 | |
289 | ASSERT_OK(Put(1, "foo", "bar")); | |
290 | // First 1MB doesn't get range synced | |
291 | ASSERT_OK(Put(1, "foo0_0", rnd_str_512kb)); | |
292 | ASSERT_OK(Put(1, "foo0_1", rnd_str_512kb)); | |
293 | ASSERT_OK(Put(1, "foo1_1", rnd_str)); | |
294 | ASSERT_OK(Put(1, "foo1_2", rnd_str)); | |
295 | ASSERT_OK(Put(1, "foo1_3", rnd_str)); | |
296 | ASSERT_OK(Put(1, "foo2", "bar")); | |
297 | ASSERT_OK(Put(1, "foo3_1", rnd_str)); | |
298 | ASSERT_OK(Put(1, "foo3_2", rnd_str)); | |
299 | ASSERT_OK(Put(1, "foo3_3", rnd_str)); | |
300 | ASSERT_OK(Put(1, "foo4", "bar")); | |
301 | dbfull()->TEST_WaitForFlushMemTable(handles_[1]); | |
302 | ||
303 | // Following writes should fail as flush failed. | |
304 | ASSERT_NOK(Put(1, "foo2", "bar3")); | |
305 | ASSERT_EQ("bar", Get(1, "foo")); | |
306 | ||
f67539c2 | 307 | ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); |
11fdf7f2 TL |
308 | ASSERT_GE(1, range_sync_called.load()); |
309 | ||
310 | ReopenWithColumnFamilies({"default", "pikachu"}, options); | |
311 | ASSERT_EQ("bar", Get(1, "foo")); | |
312 | } | |
313 | ||
314 | TEST_F(DBIOFailureTest, CompactSstRangeSyncError) { | |
315 | Options options = CurrentOptions(); | |
316 | options.env = env_; | |
317 | options.create_if_missing = true; | |
318 | options.error_if_exists = false; | |
319 | options.paranoid_checks = true; | |
320 | options.write_buffer_size = 256 * 1024 * 1024; | |
321 | options.writable_file_max_buffer_size = 128 * 1024; | |
322 | options.bytes_per_sync = 128 * 1024; | |
323 | options.level0_file_num_compaction_trigger = 2; | |
324 | options.target_file_size_base = 256 * 1024 * 1024; | |
325 | options.disable_auto_compactions = true; | |
326 | BlockBasedTableOptions table_options; | |
327 | table_options.filter_policy.reset(NewBloomFilterPolicy(10)); | |
328 | options.table_factory.reset(NewBlockBasedTableFactory(table_options)); | |
329 | DestroyAndReopen(options); | |
330 | CreateAndReopenWithCF({"pikachu"}, options); | |
331 | Status s; | |
332 | ||
333 | Random rnd(301); | |
334 | std::string rnd_str = | |
20effc67 TL |
335 | rnd.RandomString(static_cast<int>(options.bytes_per_sync / 2)); |
336 | std::string rnd_str_512kb = rnd.RandomString(512 * 1024); | |
11fdf7f2 TL |
337 | |
338 | ASSERT_OK(Put(1, "foo", "bar")); | |
339 | // First 1MB doesn't get range synced | |
340 | ASSERT_OK(Put(1, "foo0_0", rnd_str_512kb)); | |
341 | ASSERT_OK(Put(1, "foo0_1", rnd_str_512kb)); | |
342 | ASSERT_OK(Put(1, "foo1_1", rnd_str)); | |
343 | ASSERT_OK(Put(1, "foo1_2", rnd_str)); | |
344 | ASSERT_OK(Put(1, "foo1_3", rnd_str)); | |
345 | Flush(1); | |
346 | ASSERT_OK(Put(1, "foo", "bar")); | |
347 | ASSERT_OK(Put(1, "foo3_1", rnd_str)); | |
348 | ASSERT_OK(Put(1, "foo3_2", rnd_str)); | |
349 | ASSERT_OK(Put(1, "foo3_3", rnd_str)); | |
350 | ASSERT_OK(Put(1, "foo4", "bar")); | |
351 | Flush(1); | |
352 | dbfull()->TEST_WaitForFlushMemTable(handles_[1]); | |
353 | ||
354 | std::atomic<int> range_sync_called(0); | |
f67539c2 | 355 | ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( |
11fdf7f2 TL |
356 | "SpecialEnv::SStableFile::RangeSync", [&](void* arg) { |
357 | if (range_sync_called.fetch_add(1) == 0) { | |
358 | Status* st = static_cast<Status*>(arg); | |
359 | *st = Status::IOError("range sync dummy error"); | |
360 | } | |
361 | }); | |
f67539c2 | 362 | ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); |
11fdf7f2 TL |
363 | |
364 | ASSERT_OK(dbfull()->SetOptions(handles_[1], | |
365 | { | |
366 | {"disable_auto_compactions", "false"}, | |
367 | })); | |
368 | dbfull()->TEST_WaitForCompact(); | |
369 | ||
370 | // Following writes should fail as flush failed. | |
371 | ASSERT_NOK(Put(1, "foo2", "bar3")); | |
372 | ASSERT_EQ("bar", Get(1, "foo")); | |
373 | ||
f67539c2 | 374 | ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); |
11fdf7f2 TL |
375 | ASSERT_GE(1, range_sync_called.load()); |
376 | ||
377 | ReopenWithColumnFamilies({"default", "pikachu"}, options); | |
378 | ASSERT_EQ("bar", Get(1, "foo")); | |
379 | } | |
380 | ||
381 | TEST_F(DBIOFailureTest, FlushSstCloseError) { | |
382 | Options options = CurrentOptions(); | |
383 | options.env = env_; | |
384 | options.create_if_missing = true; | |
385 | options.error_if_exists = false; | |
386 | options.paranoid_checks = true; | |
387 | options.level0_file_num_compaction_trigger = 4; | |
388 | options.memtable_factory.reset(new SpecialSkipListFactory(2)); | |
389 | ||
390 | DestroyAndReopen(options); | |
391 | CreateAndReopenWithCF({"pikachu"}, options); | |
392 | Status s; | |
393 | std::atomic<int> close_called(0); | |
f67539c2 | 394 | ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( |
11fdf7f2 TL |
395 | "SpecialEnv::SStableFile::Close", [&](void* arg) { |
396 | if (close_called.fetch_add(1) == 0) { | |
397 | Status* st = static_cast<Status*>(arg); | |
398 | *st = Status::IOError("close dummy error"); | |
399 | } | |
400 | }); | |
401 | ||
f67539c2 | 402 | ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); |
11fdf7f2 TL |
403 | |
404 | ASSERT_OK(Put(1, "foo", "bar")); | |
405 | ASSERT_OK(Put(1, "foo1", "bar1")); | |
406 | ASSERT_OK(Put(1, "foo", "bar2")); | |
407 | dbfull()->TEST_WaitForFlushMemTable(handles_[1]); | |
408 | ||
409 | // Following writes should fail as flush failed. | |
410 | ASSERT_NOK(Put(1, "foo2", "bar3")); | |
411 | ASSERT_EQ("bar2", Get(1, "foo")); | |
412 | ASSERT_EQ("bar1", Get(1, "foo1")); | |
413 | ||
f67539c2 | 414 | ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); |
7c673cae | 415 | |
11fdf7f2 TL |
416 | ReopenWithColumnFamilies({"default", "pikachu"}, options); |
417 | ASSERT_EQ("bar2", Get(1, "foo")); | |
418 | ASSERT_EQ("bar1", Get(1, "foo1")); | |
419 | } | |
420 | ||
421 | TEST_F(DBIOFailureTest, CompactionSstCloseError) { | |
422 | Options options = CurrentOptions(); | |
423 | options.env = env_; | |
424 | options.create_if_missing = true; | |
425 | options.error_if_exists = false; | |
426 | options.paranoid_checks = true; | |
427 | options.level0_file_num_compaction_trigger = 2; | |
428 | options.disable_auto_compactions = true; | |
429 | ||
430 | DestroyAndReopen(options); | |
431 | CreateAndReopenWithCF({"pikachu"}, options); | |
432 | Status s; | |
433 | ||
434 | ASSERT_OK(Put(1, "foo", "bar")); | |
435 | ASSERT_OK(Put(1, "foo2", "bar")); | |
436 | Flush(1); | |
437 | ASSERT_OK(Put(1, "foo", "bar2")); | |
438 | ASSERT_OK(Put(1, "foo2", "bar")); | |
439 | Flush(1); | |
440 | ASSERT_OK(Put(1, "foo", "bar3")); | |
441 | ASSERT_OK(Put(1, "foo2", "bar")); | |
442 | Flush(1); | |
443 | dbfull()->TEST_WaitForCompact(); | |
444 | ||
445 | std::atomic<int> close_called(0); | |
f67539c2 | 446 | ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( |
11fdf7f2 TL |
447 | "SpecialEnv::SStableFile::Close", [&](void* arg) { |
448 | if (close_called.fetch_add(1) == 0) { | |
449 | Status* st = static_cast<Status*>(arg); | |
450 | *st = Status::IOError("close dummy error"); | |
451 | } | |
452 | }); | |
453 | ||
f67539c2 | 454 | ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); |
11fdf7f2 TL |
455 | ASSERT_OK(dbfull()->SetOptions(handles_[1], |
456 | { | |
457 | {"disable_auto_compactions", "false"}, | |
458 | })); | |
459 | dbfull()->TEST_WaitForCompact(); | |
460 | ||
461 | // Following writes should fail as compaction failed. | |
462 | ASSERT_NOK(Put(1, "foo2", "bar3")); | |
463 | ASSERT_EQ("bar3", Get(1, "foo")); | |
464 | ||
f67539c2 | 465 | ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); |
11fdf7f2 TL |
466 | |
467 | ReopenWithColumnFamilies({"default", "pikachu"}, options); | |
468 | ASSERT_EQ("bar3", Get(1, "foo")); | |
469 | } | |
470 | ||
471 | TEST_F(DBIOFailureTest, FlushSstSyncError) { | |
472 | Options options = CurrentOptions(); | |
473 | options.env = env_; | |
474 | options.create_if_missing = true; | |
475 | options.error_if_exists = false; | |
476 | options.paranoid_checks = true; | |
477 | options.use_fsync = false; | |
478 | options.level0_file_num_compaction_trigger = 4; | |
479 | options.memtable_factory.reset(new SpecialSkipListFactory(2)); | |
480 | ||
481 | DestroyAndReopen(options); | |
482 | CreateAndReopenWithCF({"pikachu"}, options); | |
483 | Status s; | |
484 | std::atomic<int> sync_called(0); | |
f67539c2 | 485 | ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( |
11fdf7f2 TL |
486 | "SpecialEnv::SStableFile::Sync", [&](void* arg) { |
487 | if (sync_called.fetch_add(1) == 0) { | |
488 | Status* st = static_cast<Status*>(arg); | |
489 | *st = Status::IOError("sync dummy error"); | |
490 | } | |
491 | }); | |
492 | ||
f67539c2 | 493 | ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); |
11fdf7f2 TL |
494 | |
495 | ASSERT_OK(Put(1, "foo", "bar")); | |
496 | ASSERT_OK(Put(1, "foo1", "bar1")); | |
497 | ASSERT_OK(Put(1, "foo", "bar2")); | |
498 | dbfull()->TEST_WaitForFlushMemTable(handles_[1]); | |
499 | ||
500 | // Following writes should fail as flush failed. | |
501 | ASSERT_NOK(Put(1, "foo2", "bar3")); | |
502 | ASSERT_EQ("bar2", Get(1, "foo")); | |
503 | ASSERT_EQ("bar1", Get(1, "foo1")); | |
504 | ||
f67539c2 | 505 | ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); |
11fdf7f2 TL |
506 | |
507 | ReopenWithColumnFamilies({"default", "pikachu"}, options); | |
508 | ASSERT_EQ("bar2", Get(1, "foo")); | |
509 | ASSERT_EQ("bar1", Get(1, "foo1")); | |
510 | } | |
511 | ||
512 | TEST_F(DBIOFailureTest, CompactionSstSyncError) { | |
513 | Options options = CurrentOptions(); | |
514 | options.env = env_; | |
515 | options.create_if_missing = true; | |
516 | options.error_if_exists = false; | |
517 | options.paranoid_checks = true; | |
518 | options.level0_file_num_compaction_trigger = 2; | |
519 | options.disable_auto_compactions = true; | |
520 | options.use_fsync = false; | |
521 | ||
522 | DestroyAndReopen(options); | |
523 | CreateAndReopenWithCF({"pikachu"}, options); | |
524 | Status s; | |
525 | ||
526 | ASSERT_OK(Put(1, "foo", "bar")); | |
527 | ASSERT_OK(Put(1, "foo2", "bar")); | |
528 | Flush(1); | |
529 | ASSERT_OK(Put(1, "foo", "bar2")); | |
530 | ASSERT_OK(Put(1, "foo2", "bar")); | |
531 | Flush(1); | |
532 | ASSERT_OK(Put(1, "foo", "bar3")); | |
533 | ASSERT_OK(Put(1, "foo2", "bar")); | |
534 | Flush(1); | |
535 | dbfull()->TEST_WaitForCompact(); | |
536 | ||
537 | std::atomic<int> sync_called(0); | |
f67539c2 | 538 | ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack( |
11fdf7f2 TL |
539 | "SpecialEnv::SStableFile::Sync", [&](void* arg) { |
540 | if (sync_called.fetch_add(1) == 0) { | |
541 | Status* st = static_cast<Status*>(arg); | |
542 | *st = Status::IOError("close dummy error"); | |
543 | } | |
544 | }); | |
545 | ||
f67539c2 | 546 | ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); |
11fdf7f2 TL |
547 | ASSERT_OK(dbfull()->SetOptions(handles_[1], |
548 | { | |
549 | {"disable_auto_compactions", "false"}, | |
550 | })); | |
551 | dbfull()->TEST_WaitForCompact(); | |
552 | ||
553 | // Following writes should fail as compaction failed. | |
554 | ASSERT_NOK(Put(1, "foo2", "bar3")); | |
555 | ASSERT_EQ("bar3", Get(1, "foo")); | |
556 | ||
f67539c2 | 557 | ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); |
11fdf7f2 TL |
558 | |
559 | ReopenWithColumnFamilies({"default", "pikachu"}, options); | |
560 | ASSERT_EQ("bar3", Get(1, "foo")); | |
561 | } | |
562 | #endif // !(defined NDEBUG) || !defined(OS_WIN) | |
563 | #endif // ROCKSDB_LITE | |
f67539c2 | 564 | } // namespace ROCKSDB_NAMESPACE |
7c673cae FG |
565 | |
566 | int main(int argc, char** argv) { | |
f67539c2 | 567 | ROCKSDB_NAMESPACE::port::InstallStackTraceHandler(); |
7c673cae FG |
568 | ::testing::InitGoogleTest(&argc, argv); |
569 | return RUN_ALL_TESTS(); | |
570 | } |