]> git.proxmox.com Git - ceph.git/blame - ceph/src/rocksdb/db/error_handler_fs_test.cc
projects / ceph.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
update ceph source to reef 18.1.2
[ceph.git] / ceph / src / rocksdb / db / error_handler_fs_test.cc
CommitLineData
20effc67
TL		 1// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
2// This source code is licensed under both the GPLv2 (found in the
3// COPYING file in the root directory) and Apache 2.0 License
4// (found in the LICENSE.Apache file in the root directory).
5//
6// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
7// Use of this source code is governed by a BSD-style license that can be
8// found in the LICENSE file. See the AUTHORS file for names of contributors.
9#ifndef ROCKSDB_LITE
10
11#include "db/db_test_util.h"
1e59de90 12#include "file/sst_file_manager_impl.h"
20effc67
TL		 13#include "port/stack_trace.h"
14#include "rocksdb/io_status.h"
20effc67
TL		 15#include "rocksdb/sst_file_manager.h"
16#if !defined(ROCKSDB_LITE)
17#include "test_util/sync_point.h"
18#endif
19#include "util/random.h"
20#include "utilities/fault_injection_env.h"
21#include "utilities/fault_injection_fs.h"
22
23namespace ROCKSDB_NAMESPACE {
24
25class DBErrorHandlingFSTest : public DBTestBase {
26 public:
27 DBErrorHandlingFSTest()
1e59de90 28 : DBTestBase("db_error_handling_fs_test", /*env_do_fsync=*/true) {
20effc67
TL		 29 fault_fs_.reset(new FaultInjectionTestFS(env_->GetFileSystem()));
30 fault_env_.reset(new CompositeEnvWrapper(env_, fault_fs_));
31 }
32
33 std::string GetManifestNameFromLiveFiles() {
34 std::vector<std::string> live_files;
35 uint64_t manifest_size;
36
37 Status s = dbfull()->GetLiveFiles(live_files, &manifest_size, false);
38 if (!s.ok()) {
39 return "";
40 }
41 for (auto& file : live_files) {
42 uint64_t num = 0;
43 FileType type;
44 if (ParseFileName(file, &num, &type) && type == kDescriptorFile) {
45 return file;
46 }
47 }
48 return "";
49 }
50
51 std::shared_ptr<FaultInjectionTestFS> fault_fs_;
52 std::unique_ptr<Env> fault_env_;
53};
54
55class ErrorHandlerFSListener : public EventListener {
56 public:
57 ErrorHandlerFSListener()
58 : mutex_(),
59 cv_(&mutex_),
60 no_auto_recovery_(false),
61 recovery_complete_(false),
62 file_creation_started_(false),
63 override_bg_error_(false),
64 file_count_(0),
65 fault_fs_(nullptr) {}
66 ~ErrorHandlerFSListener() {
67 file_creation_error_.PermitUncheckedError();
68 bg_error_.PermitUncheckedError();
1e59de90 69 new_bg_error_.PermitUncheckedError();
20effc67
TL		 70 }
71
72 void OnTableFileCreationStarted(
73 const TableFileCreationBriefInfo& /*ti*/) override {
74 InstrumentedMutexLock l(&mutex_);
75 file_creation_started_ = true;
76 if (file_count_ > 0) {
77 if (--file_count_ == 0) {
78 fault_fs_->SetFilesystemActive(false, file_creation_error_);
79 file_creation_error_ = IOStatus::OK();
80 }
81 }
82 cv_.SignalAll();
83 }
84
85 void OnErrorRecoveryBegin(BackgroundErrorReason /*reason*/, Status bg_error,
86 bool* auto_recovery) override {
87 bg_error.PermitUncheckedError();
88 if (*auto_recovery && no_auto_recovery_) {
89 *auto_recovery = false;
90 }
91 }
92
1e59de90 93 void OnErrorRecoveryEnd(const BackgroundErrorRecoveryInfo& info) override {
20effc67
TL		 94 InstrumentedMutexLock l(&mutex_);
95 recovery_complete_ = true;
96 cv_.SignalAll();
1e59de90 97 new_bg_error_ = info.new_bg_error;
20effc67
TL		 98 }
99
100 bool WaitForRecovery(uint64_t /*abs_time_us*/) {
101 InstrumentedMutexLock l(&mutex_);
102 while (!recovery_complete_) {
103 cv_.Wait(/*abs_time_us*/);
104 }
105 if (recovery_complete_) {
106 recovery_complete_ = false;
107 return true;
108 }
109 return false;
110 }
111
112 void WaitForTableFileCreationStarted(uint64_t /*abs_time_us*/) {
113 InstrumentedMutexLock l(&mutex_);
114 while (!file_creation_started_) {
115 cv_.Wait(/*abs_time_us*/);
116 }
117 file_creation_started_ = false;
118 }
119
120 void OnBackgroundError(BackgroundErrorReason /*reason*/,
121 Status* bg_error) override {
122 if (override_bg_error_) {
123 *bg_error = bg_error_;
124 override_bg_error_ = false;
125 }
126 }
127
128 void EnableAutoRecovery(bool enable = true) { no_auto_recovery_ = !enable; }
129
130 void OverrideBGError(Status bg_err) {
131 bg_error_ = bg_err;
132 override_bg_error_ = true;
133 }
134
135 void InjectFileCreationError(FaultInjectionTestFS* fs, int file_count,
136 IOStatus io_s) {
137 fault_fs_ = fs;
138 file_count_ = file_count;
139 file_creation_error_ = io_s;
140 }
141
1e59de90
TL		 142 Status new_bg_error() { return new_bg_error_; }
143
20effc67
TL		 144 private:
145 InstrumentedMutex mutex_;
146 InstrumentedCondVar cv_;
147 bool no_auto_recovery_;
148 bool recovery_complete_;
149 bool file_creation_started_;
150 bool override_bg_error_;
151 int file_count_;
152 IOStatus file_creation_error_;
153 Status bg_error_;
1e59de90 154 Status new_bg_error_;
20effc67
TL		 155 FaultInjectionTestFS* fault_fs_;
156};
157
158TEST_F(DBErrorHandlingFSTest, FLushWriteError) {
159 std::shared_ptr<ErrorHandlerFSListener> listener(
160 new ErrorHandlerFSListener());
161 Options options = GetDefaultOptions();
162 options.env = fault_env_.get();
163 options.create_if_missing = true;
164 options.listeners.emplace_back(listener);
1e59de90 165 options.statistics = CreateDBStatistics();
20effc67
TL		 166 Status s;
167
168 listener->EnableAutoRecovery(false);
169 DestroyAndReopen(options);
170
171 ASSERT_OK(Put(Key(0), "val"));
172 SyncPoint::GetInstance()->SetCallBack("FlushJob::Start", [&](void*) {
173 fault_fs_->SetFilesystemActive(false, IOStatus::NoSpace("Out of space"));
174 });
175 SyncPoint::GetInstance()->EnableProcessing();
176 s = Flush();
177 ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError);
178 SyncPoint::GetInstance()->DisableProcessing();
179 fault_fs_->SetFilesystemActive(true);
180 s = dbfull()->Resume();
1e59de90
TL		 181 ASSERT_OK(s);
182 ASSERT_EQ(1, options.statistics->getAndResetTickerCount(
183 ERROR_HANDLER_BG_ERROR_COUNT));
184 ASSERT_EQ(1, options.statistics->getAndResetTickerCount(
185 ERROR_HANDLER_BG_IO_ERROR_COUNT));
186 ASSERT_EQ(0, options.statistics->getAndResetTickerCount(
187 ERROR_HANDLER_BG_RETRYABLE_IO_ERROR_COUNT));
188 ASSERT_EQ(0, options.statistics->getAndResetTickerCount(
189 ERROR_HANDLER_AUTORESUME_COUNT));
190 ASSERT_EQ(0, options.statistics->getAndResetTickerCount(
191 ERROR_HANDLER_AUTORESUME_RETRY_TOTAL_COUNT));
192 ASSERT_EQ(0, options.statistics->getAndResetTickerCount(
193 ERROR_HANDLER_AUTORESUME_SUCCESS_COUNT));
20effc67
TL		 194
195 Reopen(options);
196 ASSERT_EQ("val", Get(Key(0)));
197 Destroy(options);
198}
199
1e59de90
TL		 200// All the NoSpace IOError will be handled as the regular BG Error no matter the
201// retryable flag is set of not. So the auto resume for retryable IO Error will
202// not be triggered. Also, it is mapped as hard error.
203TEST_F(DBErrorHandlingFSTest, FLushWriteNoSpaceError) {
204 std::shared_ptr<ErrorHandlerFSListener> listener(
205 new ErrorHandlerFSListener());
206 Options options = GetDefaultOptions();
207 options.env = fault_env_.get();
208 options.create_if_missing = true;
209 options.listeners.emplace_back(listener);
210 options.max_bgerror_resume_count = 2;
211 options.bgerror_resume_retry_interval = 100000; // 0.1 second
212 options.statistics = CreateDBStatistics();
213 Status s;
214
215 listener->EnableAutoRecovery(false);
216 DestroyAndReopen(options);
217
218 IOStatus error_msg = IOStatus::NoSpace("Retryable IO Error");
219 error_msg.SetRetryable(true);
220
221 ASSERT_OK(Put(Key(1), "val1"));
222 SyncPoint::GetInstance()->SetCallBack(
223 "BuildTable:BeforeFinishBuildTable",
224 [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); });
225 SyncPoint::GetInstance()->EnableProcessing();
226 s = Flush();
227 ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError);
228 SyncPoint::GetInstance()->DisableProcessing();
229 fault_fs_->SetFilesystemActive(true);
230 s = dbfull()->Resume();
231 ASSERT_OK(s);
232 ASSERT_EQ(1, options.statistics->getAndResetTickerCount(
233 ERROR_HANDLER_BG_ERROR_COUNT));
234 ASSERT_EQ(1, options.statistics->getAndResetTickerCount(
235 ERROR_HANDLER_BG_IO_ERROR_COUNT));
236 ASSERT_EQ(0, options.statistics->getAndResetTickerCount(
237 ERROR_HANDLER_BG_RETRYABLE_IO_ERROR_COUNT));
238 ASSERT_EQ(0, options.statistics->getAndResetTickerCount(
239 ERROR_HANDLER_AUTORESUME_COUNT));
240 ASSERT_EQ(0, options.statistics->getAndResetTickerCount(
241 ERROR_HANDLER_AUTORESUME_RETRY_TOTAL_COUNT));
242 ASSERT_EQ(0, options.statistics->getAndResetTickerCount(
243 ERROR_HANDLER_AUTORESUME_SUCCESS_COUNT));
244 Destroy(options);
245}
246
247TEST_F(DBErrorHandlingFSTest, FLushWriteRetryableError) {
20effc67
TL		 248 std::shared_ptr<ErrorHandlerFSListener> listener(
249 new ErrorHandlerFSListener());
250 Options options = GetDefaultOptions();
251 options.env = fault_env_.get();
252 options.create_if_missing = true;
253 options.listeners.emplace_back(listener);
254 options.max_bgerror_resume_count = 0;
1e59de90 255 options.statistics = CreateDBStatistics();
20effc67
TL		 256 Status s;
257
258 listener->EnableAutoRecovery(false);
259 DestroyAndReopen(options);
260
261 IOStatus error_msg = IOStatus::IOError("Retryable IO Error");
262 error_msg.SetRetryable(true);
263
264 ASSERT_OK(Put(Key(1), "val1"));
265 SyncPoint::GetInstance()->SetCallBack(
266 "BuildTable:BeforeFinishBuildTable",
267 [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); });
268 SyncPoint::GetInstance()->EnableProcessing();
269 s = Flush();
1e59de90 270 ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError);
20effc67
TL		 271 SyncPoint::GetInstance()->DisableProcessing();
272 fault_fs_->SetFilesystemActive(true);
273 s = dbfull()->Resume();
274 ASSERT_OK(s);
1e59de90
TL		 275 ASSERT_EQ(1, options.statistics->getAndResetTickerCount(
276 ERROR_HANDLER_BG_ERROR_COUNT));
277 ASSERT_EQ(1, options.statistics->getAndResetTickerCount(
278 ERROR_HANDLER_BG_IO_ERROR_COUNT));
279 ASSERT_EQ(1, options.statistics->getAndResetTickerCount(
280 ERROR_HANDLER_BG_RETRYABLE_IO_ERROR_COUNT));
281 ASSERT_EQ(0, options.statistics->getAndResetTickerCount(
282 ERROR_HANDLER_AUTORESUME_COUNT));
283 ASSERT_EQ(0, options.statistics->getAndResetTickerCount(
284 ERROR_HANDLER_AUTORESUME_RETRY_TOTAL_COUNT));
285 ASSERT_EQ(0, options.statistics->getAndResetTickerCount(
286 ERROR_HANDLER_AUTORESUME_SUCCESS_COUNT));
20effc67
TL		 287 Reopen(options);
288 ASSERT_EQ("val1", Get(Key(1)));
289
290 ASSERT_OK(Put(Key(2), "val2"));
291 SyncPoint::GetInstance()->SetCallBack(
292 "BuildTable:BeforeSyncTable",
293 [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); });
294 SyncPoint::GetInstance()->EnableProcessing();
295 s = Flush();
1e59de90 296 ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError);
20effc67
TL		 297 SyncPoint::GetInstance()->DisableProcessing();
298 fault_fs_->SetFilesystemActive(true);
299 s = dbfull()->Resume();
300 ASSERT_OK(s);
301 Reopen(options);
302 ASSERT_EQ("val2", Get(Key(2)));
303
304 ASSERT_OK(Put(Key(3), "val3"));
305 SyncPoint::GetInstance()->SetCallBack(
306 "BuildTable:BeforeCloseTableFile",
307 [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); });
308 SyncPoint::GetInstance()->EnableProcessing();
309 s = Flush();
1e59de90
TL		 310 ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError);
311 SyncPoint::GetInstance()->DisableProcessing();
312 fault_fs_->SetFilesystemActive(true);
313 s = dbfull()->Resume();
314 ASSERT_OK(s);
315 Reopen(options);
316 ASSERT_EQ("val3", Get(Key(3)));
317
318 Destroy(options);
319}
320
321TEST_F(DBErrorHandlingFSTest, FLushWriteFileScopeError) {
322 std::shared_ptr<ErrorHandlerFSListener> listener(
323 new ErrorHandlerFSListener());
324 Options options = GetDefaultOptions();
325 options.env = fault_env_.get();
326 options.create_if_missing = true;
327 options.listeners.emplace_back(listener);
328 options.max_bgerror_resume_count = 0;
329 Status s;
330
331 listener->EnableAutoRecovery(false);
332 DestroyAndReopen(options);
333
334 IOStatus error_msg = IOStatus::IOError("File Scope Data Loss Error");
335 error_msg.SetDataLoss(true);
336 error_msg.SetScope(
337 ROCKSDB_NAMESPACE::IOStatus::IOErrorScope::kIOErrorScopeFile);
338 error_msg.SetRetryable(false);
339
340 ASSERT_OK(Put(Key(1), "val1"));
341 SyncPoint::GetInstance()->SetCallBack(
342 "BuildTable:BeforeFinishBuildTable",
343 [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); });
344 SyncPoint::GetInstance()->EnableProcessing();
345 s = Flush();
346 ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError);
347 SyncPoint::GetInstance()->DisableProcessing();
348 fault_fs_->SetFilesystemActive(true);
349 s = dbfull()->Resume();
350 ASSERT_OK(s);
351 Reopen(options);
352 ASSERT_EQ("val1", Get(Key(1)));
353
354 ASSERT_OK(Put(Key(2), "val2"));
355 SyncPoint::GetInstance()->SetCallBack(
356 "BuildTable:BeforeSyncTable",
357 [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); });
358 SyncPoint::GetInstance()->EnableProcessing();
359 s = Flush();
360 ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError);
361 SyncPoint::GetInstance()->DisableProcessing();
362 fault_fs_->SetFilesystemActive(true);
363 s = dbfull()->Resume();
364 ASSERT_OK(s);
365 Reopen(options);
366 ASSERT_EQ("val2", Get(Key(2)));
367
368 ASSERT_OK(Put(Key(3), "val3"));
369 SyncPoint::GetInstance()->SetCallBack(
370 "BuildTable:BeforeCloseTableFile",
371 [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); });
372 SyncPoint::GetInstance()->EnableProcessing();
373 s = Flush();
374 ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError);
375 SyncPoint::GetInstance()->DisableProcessing();
376 fault_fs_->SetFilesystemActive(true);
377 s = dbfull()->Resume();
378 ASSERT_OK(s);
379 Reopen(options);
380 ASSERT_EQ("val3", Get(Key(3)));
381
382 // not file scope, but retyrable set
383 error_msg.SetDataLoss(false);
384 error_msg.SetScope(
385 ROCKSDB_NAMESPACE::IOStatus::IOErrorScope::kIOErrorScopeFileSystem);
386 error_msg.SetRetryable(true);
387
388 ASSERT_OK(Put(Key(3), "val3"));
389 SyncPoint::GetInstance()->SetCallBack(
390 "BuildTable:BeforeCloseTableFile",
391 [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); });
392 SyncPoint::GetInstance()->EnableProcessing();
393 s = Flush();
394 ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError);
20effc67
TL		 395 SyncPoint::GetInstance()->DisableProcessing();
396 fault_fs_->SetFilesystemActive(true);
397 s = dbfull()->Resume();
398 ASSERT_OK(s);
399 Reopen(options);
400 ASSERT_EQ("val3", Get(Key(3)));
401
402 Destroy(options);
403}
404
1e59de90
TL		 405TEST_F(DBErrorHandlingFSTest, FLushWALWriteRetryableError) {
406 std::shared_ptr<ErrorHandlerFSListener> listener(
407 new ErrorHandlerFSListener());
408 Options options = GetDefaultOptions();
409 options.env = fault_env_.get();
410 options.create_if_missing = true;
411 options.listeners.emplace_back(listener);
412 options.max_bgerror_resume_count = 0;
413 Status s;
414
415 IOStatus error_msg = IOStatus::IOError("Retryable IO Error");
416 error_msg.SetRetryable(true);
417
418 listener->EnableAutoRecovery(false);
419 SyncPoint::GetInstance()->SetCallBack(
420 "DBImpl::SyncClosedLogs:Start",
421 [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); });
422 SyncPoint::GetInstance()->EnableProcessing();
423
424 CreateAndReopenWithCF({"pikachu, sdfsdfsdf"}, options);
425
426 WriteOptions wo = WriteOptions();
427 wo.disableWAL = false;
428 ASSERT_OK(Put(Key(1), "val1", wo));
429
430 s = Flush();
431 ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError);
432 SyncPoint::GetInstance()->DisableProcessing();
433 fault_fs_->SetFilesystemActive(true);
434 auto cfh = dbfull()->GetColumnFamilyHandle(1);
435 s = dbfull()->DropColumnFamily(cfh);
436
437 s = dbfull()->Resume();
438 ASSERT_OK(s);
439 ASSERT_EQ("val1", Get(Key(1)));
440 ASSERT_OK(Put(Key(3), "val3", wo));
441 ASSERT_EQ("val3", Get(Key(3)));
442 s = Flush();
443 ASSERT_OK(s);
444 ASSERT_EQ("val3", Get(Key(3)));
445
446 Destroy(options);
447}
448
449TEST_F(DBErrorHandlingFSTest, FLushWALAtomicWriteRetryableError) {
450 std::shared_ptr<ErrorHandlerFSListener> listener(
451 new ErrorHandlerFSListener());
452 Options options = GetDefaultOptions();
453 options.env = fault_env_.get();
454 options.create_if_missing = true;
455 options.listeners.emplace_back(listener);
456 options.max_bgerror_resume_count = 0;
457 options.atomic_flush = true;
458 Status s;
459
460 IOStatus error_msg = IOStatus::IOError("Retryable IO Error");
461 error_msg.SetRetryable(true);
462
463 listener->EnableAutoRecovery(false);
464 SyncPoint::GetInstance()->SetCallBack(
465 "DBImpl::SyncClosedLogs:Start",
466 [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); });
467 SyncPoint::GetInstance()->EnableProcessing();
468
469 CreateAndReopenWithCF({"pikachu, sdfsdfsdf"}, options);
470
471 WriteOptions wo = WriteOptions();
472 wo.disableWAL = false;
473 ASSERT_OK(Put(Key(1), "val1", wo));
474
475 s = Flush();
476 ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError);
477 SyncPoint::GetInstance()->DisableProcessing();
478 fault_fs_->SetFilesystemActive(true);
479 auto cfh = dbfull()->GetColumnFamilyHandle(1);
480 s = dbfull()->DropColumnFamily(cfh);
481
482 s = dbfull()->Resume();
483 ASSERT_OK(s);
484 ASSERT_EQ("val1", Get(Key(1)));
485 ASSERT_OK(Put(Key(3), "val3", wo));
486 ASSERT_EQ("val3", Get(Key(3)));
487 s = Flush();
488 ASSERT_OK(s);
489 ASSERT_EQ("val3", Get(Key(3)));
490
491 Destroy(options);
492}
493
494// The flush error is injected before we finish the table build
20effc67
TL		 495TEST_F(DBErrorHandlingFSTest, FLushWritNoWALRetryableError1) {
496 std::shared_ptr<ErrorHandlerFSListener> listener(
497 new ErrorHandlerFSListener());
498 Options options = GetDefaultOptions();
499 options.env = fault_env_.get();
500 options.create_if_missing = true;
501 options.listeners.emplace_back(listener);
502 options.max_bgerror_resume_count = 0;
1e59de90 503 options.statistics = CreateDBStatistics();
20effc67
TL		 504 Status s;
505
506 listener->EnableAutoRecovery(false);
507 DestroyAndReopen(options);
508
509 IOStatus error_msg = IOStatus::IOError("Retryable IO Error");
510 error_msg.SetRetryable(true);
511
512 WriteOptions wo = WriteOptions();
513 wo.disableWAL = true;
514 ASSERT_OK(Put(Key(1), "val1", wo));
515 SyncPoint::GetInstance()->SetCallBack(
516 "BuildTable:BeforeFinishBuildTable",
517 [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); });
518 SyncPoint::GetInstance()->EnableProcessing();
519 s = Flush();
520 ASSERT_OK(Put(Key(2), "val2", wo));
521 ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError);
522 ASSERT_EQ("val2", Get(Key(2)));
523 SyncPoint::GetInstance()->DisableProcessing();
524 fault_fs_->SetFilesystemActive(true);
525 s = dbfull()->Resume();
1e59de90 526 ASSERT_OK(s);
20effc67
TL		 527 ASSERT_EQ("val1", Get(Key(1)));
528 ASSERT_EQ("val2", Get(Key(2)));
529 ASSERT_OK(Put(Key(3), "val3", wo));
530 ASSERT_EQ("val3", Get(Key(3)));
531 s = Flush();
532 ASSERT_OK(s);
533 ASSERT_EQ("val3", Get(Key(3)));
1e59de90
TL		 534 ASSERT_EQ(1, options.statistics->getAndResetTickerCount(
535 ERROR_HANDLER_BG_ERROR_COUNT));
536 ASSERT_EQ(1, options.statistics->getAndResetTickerCount(
537 ERROR_HANDLER_BG_IO_ERROR_COUNT));
538 ASSERT_EQ(1, options.statistics->getAndResetTickerCount(
539 ERROR_HANDLER_BG_RETRYABLE_IO_ERROR_COUNT));
540 ASSERT_EQ(0, options.statistics->getAndResetTickerCount(
541 ERROR_HANDLER_AUTORESUME_COUNT));
542 ASSERT_EQ(0, options.statistics->getAndResetTickerCount(
543 ERROR_HANDLER_AUTORESUME_RETRY_TOTAL_COUNT));
544 ASSERT_EQ(0, options.statistics->getAndResetTickerCount(
545 ERROR_HANDLER_AUTORESUME_SUCCESS_COUNT));
20effc67
TL		 546
547 Destroy(options);
548}
549
1e59de90
TL		 550// The retryable IO error is injected before we sync table
551TEST_F(DBErrorHandlingFSTest, FLushWriteNoWALRetryableError2) {
20effc67
TL		 552 std::shared_ptr<ErrorHandlerFSListener> listener(
553 new ErrorHandlerFSListener());
554 Options options = GetDefaultOptions();
555 options.env = fault_env_.get();
556 options.create_if_missing = true;
557 options.listeners.emplace_back(listener);
558 options.max_bgerror_resume_count = 0;
559 Status s;
560
561 listener->EnableAutoRecovery(false);
562 DestroyAndReopen(options);
563
564 IOStatus error_msg = IOStatus::IOError("Retryable IO Error");
565 error_msg.SetRetryable(true);
566
567 WriteOptions wo = WriteOptions();
568 wo.disableWAL = true;
569
570 ASSERT_OK(Put(Key(1), "val1", wo));
571 SyncPoint::GetInstance()->SetCallBack(
572 "BuildTable:BeforeSyncTable",
573 [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); });
574 SyncPoint::GetInstance()->EnableProcessing();
575 s = Flush();
576 ASSERT_OK(Put(Key(2), "val2", wo));
577 ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError);
578 ASSERT_EQ("val2", Get(Key(2)));
579 SyncPoint::GetInstance()->DisableProcessing();
580 fault_fs_->SetFilesystemActive(true);
581 s = dbfull()->Resume();
1e59de90 582 ASSERT_OK(s);
20effc67
TL		 583 ASSERT_EQ("val1", Get(Key(1)));
584 ASSERT_EQ("val2", Get(Key(2)));
585 ASSERT_OK(Put(Key(3), "val3", wo));
586 ASSERT_EQ("val3", Get(Key(3)));
587 s = Flush();
588 ASSERT_OK(s);
589 ASSERT_EQ("val3", Get(Key(3)));
590
591 Destroy(options);
592}
593
1e59de90
TL		 594// The retryable IO error is injected before we close the table file
595TEST_F(DBErrorHandlingFSTest, FLushWriteNoWALRetryableError3) {
20effc67
TL		 596 std::shared_ptr<ErrorHandlerFSListener> listener(
597 new ErrorHandlerFSListener());
598 Options options = GetDefaultOptions();
599 options.env = fault_env_.get();
600 options.create_if_missing = true;
601 options.listeners.emplace_back(listener);
602 options.max_bgerror_resume_count = 0;
603 Status s;
604
605 listener->EnableAutoRecovery(false);
606 DestroyAndReopen(options);
607
608 IOStatus error_msg = IOStatus::IOError("Retryable IO Error");
609 error_msg.SetRetryable(true);
610
611 WriteOptions wo = WriteOptions();
612 wo.disableWAL = true;
613
614 ASSERT_OK(Put(Key(1), "val1", wo));
615 SyncPoint::GetInstance()->SetCallBack(
616 "BuildTable:BeforeCloseTableFile",
617 [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); });
618 SyncPoint::GetInstance()->EnableProcessing();
619 s = Flush();
620 ASSERT_OK(Put(Key(2), "val2", wo));
621 ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError);
622 ASSERT_EQ("val2", Get(Key(2)));
623 SyncPoint::GetInstance()->DisableProcessing();
624 fault_fs_->SetFilesystemActive(true);
625 s = dbfull()->Resume();
1e59de90 626 ASSERT_OK(s);
20effc67
TL		 627 ASSERT_EQ("val1", Get(Key(1)));
628 ASSERT_EQ("val2", Get(Key(2)));
629 ASSERT_OK(Put(Key(3), "val3", wo));
630 ASSERT_EQ("val3", Get(Key(3)));
631 s = Flush();
632 ASSERT_OK(s);
633 ASSERT_EQ("val3", Get(Key(3)));
634
635 Destroy(options);
636}
637
638TEST_F(DBErrorHandlingFSTest, ManifestWriteError) {
639 std::shared_ptr<ErrorHandlerFSListener> listener(
640 new ErrorHandlerFSListener());
641 Options options = GetDefaultOptions();
642 options.env = fault_env_.get();
643 options.create_if_missing = true;
644 options.listeners.emplace_back(listener);
645 Status s;
646 std::string old_manifest;
647 std::string new_manifest;
648
649 listener->EnableAutoRecovery(false);
650 DestroyAndReopen(options);
651 old_manifest = GetManifestNameFromLiveFiles();
652
653 ASSERT_OK(Put(Key(0), "val"));
654 ASSERT_OK(Flush());
655 ASSERT_OK(Put(Key(1), "val"));
656 SyncPoint::GetInstance()->SetCallBack(
657 "VersionSet::LogAndApply:WriteManifest", [&](void*) {
658 fault_fs_->SetFilesystemActive(false,
659 IOStatus::NoSpace("Out of space"));
660 });
661 SyncPoint::GetInstance()->EnableProcessing();
662 s = Flush();
663 ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError);
664 SyncPoint::GetInstance()->ClearAllCallBacks();
665 SyncPoint::GetInstance()->DisableProcessing();
666 fault_fs_->SetFilesystemActive(true);
667 s = dbfull()->Resume();
1e59de90 668 ASSERT_OK(s);
20effc67
TL		 669
670 new_manifest = GetManifestNameFromLiveFiles();
671 ASSERT_NE(new_manifest, old_manifest);
672
673 Reopen(options);
674 ASSERT_EQ("val", Get(Key(0)));
675 ASSERT_EQ("val", Get(Key(1)));
676 Close();
677}
678
679TEST_F(DBErrorHandlingFSTest, ManifestWriteRetryableError) {
680 std::shared_ptr<ErrorHandlerFSListener> listener(
681 new ErrorHandlerFSListener());
682 Options options = GetDefaultOptions();
683 options.env = fault_env_.get();
684 options.create_if_missing = true;
685 options.listeners.emplace_back(listener);
686 options.max_bgerror_resume_count = 0;
687 Status s;
688 std::string old_manifest;
689 std::string new_manifest;
690
691 listener->EnableAutoRecovery(false);
692 DestroyAndReopen(options);
693 old_manifest = GetManifestNameFromLiveFiles();
694
695 IOStatus error_msg = IOStatus::IOError("Retryable IO Error");
696 error_msg.SetRetryable(true);
697
698 ASSERT_OK(Put(Key(0), "val"));
699 ASSERT_OK(Flush());
700 ASSERT_OK(Put(Key(1), "val"));
701 SyncPoint::GetInstance()->SetCallBack(
702 "VersionSet::LogAndApply:WriteManifest",
703 [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); });
704 SyncPoint::GetInstance()->EnableProcessing();
705 s = Flush();
1e59de90
TL		 706 ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError);
707 SyncPoint::GetInstance()->ClearAllCallBacks();
708 SyncPoint::GetInstance()->DisableProcessing();
709 fault_fs_->SetFilesystemActive(true);
710 s = dbfull()->Resume();
711 ASSERT_OK(s);
712
713 new_manifest = GetManifestNameFromLiveFiles();
714 ASSERT_NE(new_manifest, old_manifest);
715
716 Reopen(options);
717 ASSERT_EQ("val", Get(Key(0)));
718 ASSERT_EQ("val", Get(Key(1)));
719 Close();
720}
721
722TEST_F(DBErrorHandlingFSTest, ManifestWriteFileScopeError) {
723 std::shared_ptr<ErrorHandlerFSListener> listener(
724 new ErrorHandlerFSListener());
725 Options options = GetDefaultOptions();
726 options.env = fault_env_.get();
727 options.create_if_missing = true;
728 options.listeners.emplace_back(listener);
729 options.max_bgerror_resume_count = 0;
730 Status s;
731 std::string old_manifest;
732 std::string new_manifest;
733
734 listener->EnableAutoRecovery(false);
735 DestroyAndReopen(options);
736 old_manifest = GetManifestNameFromLiveFiles();
737
738 IOStatus error_msg = IOStatus::IOError("File Scope Data Loss Error");
739 error_msg.SetDataLoss(true);
740 error_msg.SetScope(
741 ROCKSDB_NAMESPACE::IOStatus::IOErrorScope::kIOErrorScopeFile);
742 error_msg.SetRetryable(false);
743
744 ASSERT_OK(Put(Key(0), "val"));
745 ASSERT_OK(Flush());
746 ASSERT_OK(Put(Key(1), "val"));
747 SyncPoint::GetInstance()->SetCallBack(
748 "VersionSet::LogAndApply:WriteManifest",
749 [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); });
750 SyncPoint::GetInstance()->EnableProcessing();
751 s = Flush();
752 ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError);
20effc67
TL		 753 SyncPoint::GetInstance()->ClearAllCallBacks();
754 SyncPoint::GetInstance()->DisableProcessing();
755 fault_fs_->SetFilesystemActive(true);
756 s = dbfull()->Resume();
1e59de90
TL		 757 ASSERT_OK(s);
758
759 new_manifest = GetManifestNameFromLiveFiles();
760 ASSERT_NE(new_manifest, old_manifest);
761
762 Reopen(options);
763 ASSERT_EQ("val", Get(Key(0)));
764 ASSERT_EQ("val", Get(Key(1)));
765 Close();
766}
767
768TEST_F(DBErrorHandlingFSTest, ManifestWriteNoWALRetryableError) {
769 std::shared_ptr<ErrorHandlerFSListener> listener(
770 new ErrorHandlerFSListener());
771 Options options = GetDefaultOptions();
772 options.env = fault_env_.get();
773 options.create_if_missing = true;
774 options.listeners.emplace_back(listener);
775 options.max_bgerror_resume_count = 0;
776 Status s;
777 std::string old_manifest;
778 std::string new_manifest;
779
780 listener->EnableAutoRecovery(false);
781 DestroyAndReopen(options);
782 old_manifest = GetManifestNameFromLiveFiles();
783
784 IOStatus error_msg = IOStatus::IOError("Retryable IO Error");
785 error_msg.SetRetryable(true);
786
787 WriteOptions wo = WriteOptions();
788 wo.disableWAL = true;
789 ASSERT_OK(Put(Key(0), "val", wo));
790 ASSERT_OK(Flush());
791 ASSERT_OK(Put(Key(1), "val", wo));
792 SyncPoint::GetInstance()->SetCallBack(
793 "VersionSet::LogAndApply:WriteManifest",
794 [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); });
795 SyncPoint::GetInstance()->EnableProcessing();
796 s = Flush();
797 ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError);
798 SyncPoint::GetInstance()->ClearAllCallBacks();
799 SyncPoint::GetInstance()->DisableProcessing();
800 fault_fs_->SetFilesystemActive(true);
801 s = dbfull()->Resume();
802 ASSERT_OK(s);
20effc67
TL		 803
804 new_manifest = GetManifestNameFromLiveFiles();
805 ASSERT_NE(new_manifest, old_manifest);
806
807 Reopen(options);
808 ASSERT_EQ("val", Get(Key(0)));
809 ASSERT_EQ("val", Get(Key(1)));
810 Close();
811}
812
813TEST_F(DBErrorHandlingFSTest, DoubleManifestWriteError) {
814 std::shared_ptr<ErrorHandlerFSListener> listener(
815 new ErrorHandlerFSListener());
816 Options options = GetDefaultOptions();
817 options.env = fault_env_.get();
818 options.create_if_missing = true;
819 options.listeners.emplace_back(listener);
820 Status s;
821 std::string old_manifest;
822 std::string new_manifest;
823
824 listener->EnableAutoRecovery(false);
825 DestroyAndReopen(options);
826 old_manifest = GetManifestNameFromLiveFiles();
827
828 ASSERT_OK(Put(Key(0), "val"));
829 ASSERT_OK(Flush());
830 ASSERT_OK(Put(Key(1), "val"));
831 SyncPoint::GetInstance()->SetCallBack(
832 "VersionSet::LogAndApply:WriteManifest", [&](void*) {
833 fault_fs_->SetFilesystemActive(false,
834 IOStatus::NoSpace("Out of space"));
835 });
836 SyncPoint::GetInstance()->EnableProcessing();
837 s = Flush();
838 ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError);
839 fault_fs_->SetFilesystemActive(true);
840
841 // This Resume() will attempt to create a new manifest file and fail again
842 s = dbfull()->Resume();
843 ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError);
844 fault_fs_->SetFilesystemActive(true);
845 SyncPoint::GetInstance()->ClearAllCallBacks();
846 SyncPoint::GetInstance()->DisableProcessing();
847
848 // A successful Resume() will create a new manifest file
849 s = dbfull()->Resume();
1e59de90 850 ASSERT_OK(s);
20effc67
TL		 851
852 new_manifest = GetManifestNameFromLiveFiles();
853 ASSERT_NE(new_manifest, old_manifest);
854
855 Reopen(options);
856 ASSERT_EQ("val", Get(Key(0)));
857 ASSERT_EQ("val", Get(Key(1)));
858 Close();
859}
860
861TEST_F(DBErrorHandlingFSTest, CompactionManifestWriteError) {
862 if (mem_env_ != nullptr) {
863 ROCKSDB_GTEST_SKIP("Test requires non-mock environment");
864 return;
865 }
866 std::shared_ptr<ErrorHandlerFSListener> listener(
867 new ErrorHandlerFSListener());
868 Options options = GetDefaultOptions();
869 options.env = fault_env_.get();
870 options.create_if_missing = true;
871 options.level0_file_num_compaction_trigger = 2;
872 options.listeners.emplace_back(listener);
873 Status s;
874 std::string old_manifest;
875 std::string new_manifest;
876 std::atomic<bool> fail_manifest(false);
877 DestroyAndReopen(options);
878 old_manifest = GetManifestNameFromLiveFiles();
879
880 ASSERT_OK(Put(Key(0), "val"));
881 ASSERT_OK(Put(Key(2), "val"));
882 s = Flush();
1e59de90 883 ASSERT_OK(s);
20effc67
TL		 884
885 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency(
886 // Wait for flush of 2nd L0 file before starting compaction
887 {{"DBImpl::FlushMemTable:FlushMemTableFinished",
888 "BackgroundCallCompaction:0"},
889 // Wait for compaction to detect manifest write error
890 {"BackgroundCallCompaction:1", "CompactionManifestWriteError:0"},
891 // Make compaction thread wait for error to be cleared
892 {"CompactionManifestWriteError:1",
893 "DBImpl::BackgroundCallCompaction:FoundObsoleteFiles"},
894 // Wait for DB instance to clear bg_error before calling
895 // TEST_WaitForCompact
896 {"SstFileManagerImpl::ErrorCleared", "CompactionManifestWriteError:2"}});
897 // trigger manifest write failure in compaction thread
898 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
899 "BackgroundCallCompaction:0", [&](void*) { fail_manifest.store(true); });
900 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
901 "VersionSet::LogAndApply:WriteManifest", [&](void*) {
902 if (fail_manifest.load()) {
903 fault_fs_->SetFilesystemActive(false,
904 IOStatus::NoSpace("Out of space"));
905 }
906 });
907 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
908
909 ASSERT_OK(Put(Key(1), "val"));
910 // This Flush will trigger a compaction, which will fail when appending to
911 // the manifest
912 s = Flush();
1e59de90 913 ASSERT_OK(s);
20effc67
TL		 914
915 TEST_SYNC_POINT("CompactionManifestWriteError:0");
916 // Clear all errors so when the compaction is retried, it will succeed
917 fault_fs_->SetFilesystemActive(true);
918 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
919 TEST_SYNC_POINT("CompactionManifestWriteError:1");
920 TEST_SYNC_POINT("CompactionManifestWriteError:2");
921
922 s = dbfull()->TEST_WaitForCompact();
923 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
1e59de90 924 ASSERT_OK(s);
20effc67
TL		 925
926 new_manifest = GetManifestNameFromLiveFiles();
927 ASSERT_NE(new_manifest, old_manifest);
928 Reopen(options);
929 ASSERT_EQ("val", Get(Key(0)));
930 ASSERT_EQ("val", Get(Key(1)));
931 ASSERT_EQ("val", Get(Key(2)));
932 Close();
933}
934
935TEST_F(DBErrorHandlingFSTest, CompactionManifestWriteRetryableError) {
936 std::shared_ptr<ErrorHandlerFSListener> listener(
937 new ErrorHandlerFSListener());
938 Options options = GetDefaultOptions();
939 options.env = fault_env_.get();
940 options.create_if_missing = true;
941 options.level0_file_num_compaction_trigger = 2;
942 options.listeners.emplace_back(listener);
943 options.max_bgerror_resume_count = 0;
944 Status s;
945 std::string old_manifest;
946 std::string new_manifest;
947 std::atomic<bool> fail_manifest(false);
948 DestroyAndReopen(options);
949 old_manifest = GetManifestNameFromLiveFiles();
950
951 IOStatus error_msg = IOStatus::IOError("Retryable IO Error");
952 error_msg.SetRetryable(true);
953
954 ASSERT_OK(Put(Key(0), "val"));
955 ASSERT_OK(Put(Key(2), "val"));
956 s = Flush();
1e59de90 957 ASSERT_OK(s);
20effc67
TL		 958
959 listener->OverrideBGError(Status(error_msg, Status::Severity::kHardError));
960 listener->EnableAutoRecovery(false);
961 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency(
962 // Wait for flush of 2nd L0 file before starting compaction
963 {{"DBImpl::FlushMemTable:FlushMemTableFinished",
964 "BackgroundCallCompaction:0"},
965 // Wait for compaction to detect manifest write error
966 {"BackgroundCallCompaction:1", "CompactionManifestWriteError:0"},
967 // Make compaction thread wait for error to be cleared
968 {"CompactionManifestWriteError:1",
969 "DBImpl::BackgroundCallCompaction:FoundObsoleteFiles"}});
970 // trigger manifest write failure in compaction thread
971 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
972 "BackgroundCallCompaction:0", [&](void*) { fail_manifest.store(true); });
973 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
974 "VersionSet::LogAndApply:WriteManifest", [&](void*) {
975 if (fail_manifest.load()) {
976 fault_fs_->SetFilesystemActive(false, error_msg);
977 }
978 });
979 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
980
981 ASSERT_OK(Put(Key(1), "val"));
982 s = Flush();
1e59de90 983 ASSERT_OK(s);
20effc67
TL		 984
985 TEST_SYNC_POINT("CompactionManifestWriteError:0");
986 TEST_SYNC_POINT("CompactionManifestWriteError:1");
987
988 s = dbfull()->TEST_WaitForCompact();
989 ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError);
990
991 fault_fs_->SetFilesystemActive(true);
992 SyncPoint::GetInstance()->ClearAllCallBacks();
993 SyncPoint::GetInstance()->DisableProcessing();
994 s = dbfull()->Resume();
1e59de90 995 ASSERT_OK(s);
20effc67
TL		 996
997 new_manifest = GetManifestNameFromLiveFiles();
998 ASSERT_NE(new_manifest, old_manifest);
999
1000 Reopen(options);
1001 ASSERT_EQ("val", Get(Key(0)));
1002 ASSERT_EQ("val", Get(Key(1)));
1003 ASSERT_EQ("val", Get(Key(2)));
1004 Close();
1005}
1006
1007TEST_F(DBErrorHandlingFSTest, CompactionWriteError) {
1008 std::shared_ptr<ErrorHandlerFSListener> listener(
1009 new ErrorHandlerFSListener());
1010 Options options = GetDefaultOptions();
1011 options.env = fault_env_.get();
1012 options.create_if_missing = true;
1013 options.level0_file_num_compaction_trigger = 2;
1014 options.listeners.emplace_back(listener);
1015 Status s;
1016 DestroyAndReopen(options);
1017
1018 ASSERT_OK(Put(Key(0), "va;"));
1019 ASSERT_OK(Put(Key(2), "va;"));
1020 s = Flush();
1e59de90 1021 ASSERT_OK(s);
20effc67
TL		 1022
1023 listener->OverrideBGError(
1024 Status(Status::NoSpace(), Status::Severity::kHardError));
1025 listener->EnableAutoRecovery(false);
1026 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency(
1027 {{"DBImpl::FlushMemTable:FlushMemTableFinished",
1028 "BackgroundCallCompaction:0"}});
1029 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
1030 "BackgroundCallCompaction:0", [&](void*) {
1031 fault_fs_->SetFilesystemActive(false,
1032 IOStatus::NoSpace("Out of space"));
1033 });
1034 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
1035
1036 ASSERT_OK(Put(Key(1), "val"));
1037 s = Flush();
1e59de90 1038 ASSERT_OK(s);
20effc67
TL		 1039
1040 s = dbfull()->TEST_WaitForCompact();
1041 ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError);
1042
1043 fault_fs_->SetFilesystemActive(true);
1044 s = dbfull()->Resume();
1e59de90 1045 ASSERT_OK(s);
20effc67
TL		 1046 Destroy(options);
1047}
1048
1e59de90 1049TEST_F(DBErrorHandlingFSTest, DISABLED_CompactionWriteRetryableError) {
20effc67
TL		 1050 std::shared_ptr<ErrorHandlerFSListener> listener(
1051 new ErrorHandlerFSListener());
1052 Options options = GetDefaultOptions();
1053 options.env = fault_env_.get();
1054 options.create_if_missing = true;
1055 options.level0_file_num_compaction_trigger = 2;
1056 options.listeners.emplace_back(listener);
1057 options.max_bgerror_resume_count = 0;
1058 Status s;
1059 DestroyAndReopen(options);
1060
1061 IOStatus error_msg = IOStatus::IOError("Retryable IO Error");
1062 error_msg.SetRetryable(true);
1063
1064 ASSERT_OK(Put(Key(0), "va;"));
1065 ASSERT_OK(Put(Key(2), "va;"));
1066 s = Flush();
1e59de90 1067 ASSERT_OK(s);
20effc67
TL		 1068
1069 listener->OverrideBGError(Status(error_msg, Status::Severity::kHardError));
1070 listener->EnableAutoRecovery(false);
1071 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency(
1072 {{"DBImpl::FlushMemTable:FlushMemTableFinished",
1073 "BackgroundCallCompaction:0"}});
1074 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
1075 "CompactionJob::OpenCompactionOutputFile",
1076 [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); });
1e59de90
TL		 1077 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
1078 "DBImpl::BackgroundCompaction:Finish",
1079 [&](void*) { CancelAllBackgroundWork(dbfull()); });
20effc67
TL		 1080 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
1081
1082 ASSERT_OK(Put(Key(1), "val"));
1083 s = Flush();
1e59de90 1084 ASSERT_OK(s);
20effc67 1085
1e59de90
TL		 1086 s = dbfull()->TEST_GetBGError();
1087 ASSERT_OK(s);
1088 fault_fs_->SetFilesystemActive(true);
1089 SyncPoint::GetInstance()->ClearAllCallBacks();
1090 SyncPoint::GetInstance()->DisableProcessing();
1091 s = dbfull()->Resume();
1092 ASSERT_OK(s);
1093 Destroy(options);
1094}
1095
1096TEST_F(DBErrorHandlingFSTest, DISABLED_CompactionWriteFileScopeError) {
1097 std::shared_ptr<ErrorHandlerFSListener> listener(
1098 new ErrorHandlerFSListener());
1099 Options options = GetDefaultOptions();
1100 options.env = fault_env_.get();
1101 options.create_if_missing = true;
1102 options.level0_file_num_compaction_trigger = 2;
1103 options.listeners.emplace_back(listener);
1104 options.max_bgerror_resume_count = 0;
1105 Status s;
1106 DestroyAndReopen(options);
1107
1108 IOStatus error_msg = IOStatus::IOError("File Scope Data Loss Error");
1109 error_msg.SetDataLoss(true);
1110 error_msg.SetScope(
1111 ROCKSDB_NAMESPACE::IOStatus::IOErrorScope::kIOErrorScopeFile);
1112 error_msg.SetRetryable(false);
1113
1114 ASSERT_OK(Put(Key(0), "va;"));
1115 ASSERT_OK(Put(Key(2), "va;"));
1116 s = Flush();
1117 ASSERT_OK(s);
1118
1119 listener->OverrideBGError(Status(error_msg, Status::Severity::kHardError));
1120 listener->EnableAutoRecovery(false);
1121 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency(
1122 {{"DBImpl::FlushMemTable:FlushMemTableFinished",
1123 "BackgroundCallCompaction:0"}});
1124 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
1125 "CompactionJob::OpenCompactionOutputFile",
1126 [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); });
1127 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
1128 "DBImpl::BackgroundCompaction:Finish",
1129 [&](void*) { CancelAllBackgroundWork(dbfull()); });
1130 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
1131
1132 ASSERT_OK(Put(Key(1), "val"));
1133 s = Flush();
1134 ASSERT_OK(s);
1135
1136 s = dbfull()->TEST_GetBGError();
1137 ASSERT_OK(s);
20effc67
TL		 1138
1139 fault_fs_->SetFilesystemActive(true);
1140 SyncPoint::GetInstance()->ClearAllCallBacks();
1141 SyncPoint::GetInstance()->DisableProcessing();
1142 s = dbfull()->Resume();
1e59de90 1143 ASSERT_OK(s);
20effc67
TL		 1144 Destroy(options);
1145}
1146
1147TEST_F(DBErrorHandlingFSTest, CorruptionError) {
1148 Options options = GetDefaultOptions();
1149 options.env = fault_env_.get();
1150 options.create_if_missing = true;
1151 options.level0_file_num_compaction_trigger = 2;
1152 Status s;
1153 DestroyAndReopen(options);
1154
1155 ASSERT_OK(Put(Key(0), "va;"));
1156 ASSERT_OK(Put(Key(2), "va;"));
1157 s = Flush();
1e59de90 1158 ASSERT_OK(s);
20effc67
TL		 1159
1160 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency(
1161 {{"DBImpl::FlushMemTable:FlushMemTableFinished",
1162 "BackgroundCallCompaction:0"}});
1163 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
1164 "BackgroundCallCompaction:0", [&](void*) {
1165 fault_fs_->SetFilesystemActive(false,
1166 IOStatus::Corruption("Corruption"));
1167 });
1168 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
1169
1170 ASSERT_OK(Put(Key(1), "val"));
1171 s = Flush();
1e59de90 1172 ASSERT_OK(s);
20effc67
TL		 1173
1174 s = dbfull()->TEST_WaitForCompact();
1175 ASSERT_EQ(s.severity(),
1176 ROCKSDB_NAMESPACE::Status::Severity::kUnrecoverableError);
1177
1178 fault_fs_->SetFilesystemActive(true);
1179 s = dbfull()->Resume();
1e59de90 1180 ASSERT_NOK(s);
20effc67
TL		 1181 Destroy(options);
1182}
1183
1184TEST_F(DBErrorHandlingFSTest, AutoRecoverFlushError) {
1185 if (mem_env_ != nullptr) {
1186 ROCKSDB_GTEST_SKIP("Test requires non-mock environment");
1187 return;
1188 }
1189 std::shared_ptr<ErrorHandlerFSListener> listener(
1190 new ErrorHandlerFSListener());
1191 Options options = GetDefaultOptions();
1192 options.env = fault_env_.get();
1193 options.create_if_missing = true;
1194 options.listeners.emplace_back(listener);
1e59de90 1195 options.statistics = CreateDBStatistics();
20effc67
TL		 1196 Status s;
1197
1198 listener->EnableAutoRecovery();
1199 DestroyAndReopen(options);
1200
1201 ASSERT_OK(Put(Key(0), "val"));
1202 SyncPoint::GetInstance()->SetCallBack("FlushJob::Start", [&](void*) {
1203 fault_fs_->SetFilesystemActive(false, IOStatus::NoSpace("Out of space"));
1204 });
1205 SyncPoint::GetInstance()->EnableProcessing();
1206 s = Flush();
1207 ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError);
1208 SyncPoint::GetInstance()->DisableProcessing();
1209 fault_fs_->SetFilesystemActive(true);
1210 ASSERT_EQ(listener->WaitForRecovery(5000000), true);
1211
1212 s = Put(Key(1), "val");
1e59de90
TL		 1213 ASSERT_OK(s);
1214 ASSERT_EQ(1, options.statistics->getAndResetTickerCount(
1215 ERROR_HANDLER_BG_ERROR_COUNT));
1216 ASSERT_EQ(1, options.statistics->getAndResetTickerCount(
1217 ERROR_HANDLER_BG_IO_ERROR_COUNT));
1218 ASSERT_EQ(0, options.statistics->getAndResetTickerCount(
1219 ERROR_HANDLER_BG_RETRYABLE_IO_ERROR_COUNT));
1220 ASSERT_EQ(0, options.statistics->getAndResetTickerCount(
1221 ERROR_HANDLER_AUTORESUME_COUNT));
1222 ASSERT_EQ(0, options.statistics->getAndResetTickerCount(
1223 ERROR_HANDLER_AUTORESUME_RETRY_TOTAL_COUNT));
1224 ASSERT_EQ(0, options.statistics->getAndResetTickerCount(
1225 ERROR_HANDLER_AUTORESUME_SUCCESS_COUNT));
20effc67
TL		 1226
1227 Reopen(options);
1228 ASSERT_EQ("val", Get(Key(0)));
1229 ASSERT_EQ("val", Get(Key(1)));
1230 Destroy(options);
1231}
1232
1233TEST_F(DBErrorHandlingFSTest, FailRecoverFlushError) {
1234 std::shared_ptr<ErrorHandlerFSListener> listener(
1235 new ErrorHandlerFSListener());
1236 Options options = GetDefaultOptions();
1237 options.env = fault_env_.get();
1238 options.create_if_missing = true;
1239 options.listeners.emplace_back(listener);
1240 Status s;
1241
1242 listener->EnableAutoRecovery();
1243 DestroyAndReopen(options);
1244
1245 ASSERT_OK(Put(Key(0), "val"));
1246 SyncPoint::GetInstance()->SetCallBack("FlushJob::Start", [&](void*) {
1247 fault_fs_->SetFilesystemActive(false, IOStatus::NoSpace("Out of space"));
1248 });
1249 SyncPoint::GetInstance()->EnableProcessing();
1250 s = Flush();
1251 ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError);
1252 // We should be able to shutdown the database while auto recovery is going
1253 // on in the background
1254 Close();
1e59de90 1255 DestroyDB(dbname_, options).PermitUncheckedError();
20effc67
TL		 1256}
1257
1258TEST_F(DBErrorHandlingFSTest, WALWriteError) {
1259 if (mem_env_ != nullptr) {
1260 ROCKSDB_GTEST_SKIP("Test requires non-mock environment");
1261 return;
1262 }
1263 std::shared_ptr<ErrorHandlerFSListener> listener(
1264 new ErrorHandlerFSListener());
1265 Options options = GetDefaultOptions();
1266 options.env = fault_env_.get();
1267 options.create_if_missing = true;
1268 options.writable_file_max_buffer_size = 32768;
1269 options.listeners.emplace_back(listener);
1270 Status s;
1271 Random rnd(301);
1272
1273 listener->EnableAutoRecovery();
1274 DestroyAndReopen(options);
1275
1276 {
1277 WriteBatch batch;
1278
1279 for (auto i = 0; i < 100; ++i) {
1280 ASSERT_OK(batch.Put(Key(i), rnd.RandomString(1024)));
1281 }
1282
1283 WriteOptions wopts;
1284 wopts.sync = true;
1e59de90 1285 ASSERT_OK(dbfull()->Write(wopts, &batch));
20effc67
TL		 1286 };
1287
1288 {
1289 WriteBatch batch;
1290 int write_error = 0;
1291
1292 for (auto i = 100; i < 199; ++i) {
1293 ASSERT_OK(batch.Put(Key(i), rnd.RandomString(1024)));
1294 }
1295
1296 SyncPoint::GetInstance()->SetCallBack(
1297 "WritableFileWriter::Append:BeforePrepareWrite", [&](void*) {
1298 write_error++;
1299 if (write_error > 2) {
1300 fault_fs_->SetFilesystemActive(false,
1301 IOStatus::NoSpace("Out of space"));
1302 }
1303 });
1304 SyncPoint::GetInstance()->EnableProcessing();
1305 WriteOptions wopts;
1306 wopts.sync = true;
1307 s = dbfull()->Write(wopts, &batch);
1308 ASSERT_EQ(s, s.NoSpace());
1309 }
1310 SyncPoint::GetInstance()->DisableProcessing();
1e59de90
TL		 1311 // `ClearAllCallBacks()` is needed in addition to `DisableProcessing()` to
1312 // drain all callbacks. Otherwise, a pending callback in the background
1313 // could re-disable `fault_fs_` after we enable it below.
1314 SyncPoint::GetInstance()->ClearAllCallBacks();
20effc67
TL		 1315 fault_fs_->SetFilesystemActive(true);
1316 ASSERT_EQ(listener->WaitForRecovery(5000000), true);
1317 for (auto i = 0; i < 199; ++i) {
1318 if (i < 100) {
1319 ASSERT_NE(Get(Key(i)), "NOT_FOUND");
1320 } else {
1321 ASSERT_EQ(Get(Key(i)), "NOT_FOUND");
1322 }
1323 }
1324 Reopen(options);
1325 for (auto i = 0; i < 199; ++i) {
1326 if (i < 100) {
1327 ASSERT_NE(Get(Key(i)), "NOT_FOUND");
1328 } else {
1329 ASSERT_EQ(Get(Key(i)), "NOT_FOUND");
1330 }
1331 }
1332 Close();
1333}
1334
1335TEST_F(DBErrorHandlingFSTest, WALWriteRetryableError) {
1336 std::shared_ptr<ErrorHandlerFSListener> listener(
1337 new ErrorHandlerFSListener());
1338 Options options = GetDefaultOptions();
1339 options.env = fault_env_.get();
1340 options.create_if_missing = true;
1341 options.writable_file_max_buffer_size = 32768;
1342 options.listeners.emplace_back(listener);
1343 options.paranoid_checks = true;
1344 options.max_bgerror_resume_count = 0;
20effc67
TL		 1345 Random rnd(301);
1346
1347 DestroyAndReopen(options);
1348
1349 IOStatus error_msg = IOStatus::IOError("Retryable IO Error");
1350 error_msg.SetRetryable(true);
1351
1352 // For the first batch, write is successful, require sync
1353 {
1354 WriteBatch batch;
1355
1356 for (auto i = 0; i < 100; ++i) {
1357 ASSERT_OK(batch.Put(Key(i), rnd.RandomString(1024)));
1358 }
1359
1360 WriteOptions wopts;
1361 wopts.sync = true;
1e59de90 1362 ASSERT_OK(dbfull()->Write(wopts, &batch));
20effc67
TL		 1363 };
1364
1365 // For the second batch, the first 2 file Append are successful, then the
1366 // following Append fails due to file system retryable IOError.
1367 {
1368 WriteBatch batch;
1369 int write_error = 0;
1370
1371 for (auto i = 100; i < 200; ++i) {
1372 ASSERT_OK(batch.Put(Key(i), rnd.RandomString(1024)));
1373 }
1374
1375 SyncPoint::GetInstance()->SetCallBack(
1376 "WritableFileWriter::Append:BeforePrepareWrite", [&](void*) {
1377 write_error++;
1378 if (write_error > 2) {
1379 fault_fs_->SetFilesystemActive(false, error_msg);
1380 }
1381 });
1382 SyncPoint::GetInstance()->EnableProcessing();
1383 WriteOptions wopts;
1384 wopts.sync = true;
1e59de90
TL		 1385 Status s = dbfull()->Write(wopts, &batch);
1386 ASSERT_TRUE(s.IsIOError());
20effc67
TL		 1387 }
1388 fault_fs_->SetFilesystemActive(true);
1389 SyncPoint::GetInstance()->ClearAllCallBacks();
1390 SyncPoint::GetInstance()->DisableProcessing();
1391
1392 // Data in corrupted WAL are not stored
1393 for (auto i = 0; i < 199; ++i) {
1394 if (i < 100) {
1395 ASSERT_NE(Get(Key(i)), "NOT_FOUND");
1396 } else {
1397 ASSERT_EQ(Get(Key(i)), "NOT_FOUND");
1398 }
1399 }
1400
1401 // Resume and write a new batch, should be in the WAL
1e59de90 1402 ASSERT_OK(dbfull()->Resume());
20effc67
TL		 1403 {
1404 WriteBatch batch;
1405
1406 for (auto i = 200; i < 300; ++i) {
1407 ASSERT_OK(batch.Put(Key(i), rnd.RandomString(1024)));
1408 }
1409
1410 WriteOptions wopts;
1411 wopts.sync = true;
1e59de90 1412 ASSERT_OK(dbfull()->Write(wopts, &batch));
20effc67
TL		 1413 };
1414
1415 Reopen(options);
1416 for (auto i = 0; i < 300; ++i) {
1417 if (i < 100 || i >= 200) {
1418 ASSERT_NE(Get(Key(i)), "NOT_FOUND");
1419 } else {
1420 ASSERT_EQ(Get(Key(i)), "NOT_FOUND");
1421 }
1422 }
1423 Close();
1424}
1425
1426TEST_F(DBErrorHandlingFSTest, MultiCFWALWriteError) {
1427 if (mem_env_ != nullptr) {
1428 ROCKSDB_GTEST_SKIP("Test requires non-mock environment");
1429 return;
1430 }
1431 std::shared_ptr<ErrorHandlerFSListener> listener(
1432 new ErrorHandlerFSListener());
1433 Options options = GetDefaultOptions();
1434 options.env = fault_env_.get();
1435 options.create_if_missing = true;
1436 options.writable_file_max_buffer_size = 32768;
1437 options.listeners.emplace_back(listener);
20effc67
TL		 1438 Random rnd(301);
1439
1440 listener->EnableAutoRecovery();
1441 CreateAndReopenWithCF({"one", "two", "three"}, options);
1442
1443 {
1444 WriteBatch batch;
1445
1446 for (auto i = 1; i < 4; ++i) {
1447 for (auto j = 0; j < 100; ++j) {
1448 ASSERT_OK(batch.Put(handles_[i], Key(j), rnd.RandomString(1024)));
1449 }
1450 }
1451
1452 WriteOptions wopts;
1453 wopts.sync = true;
1e59de90 1454 ASSERT_OK(dbfull()->Write(wopts, &batch));
20effc67
TL		 1455 };
1456
1457 {
1458 WriteBatch batch;
1459 int write_error = 0;
1460
1461 // Write to one CF
1462 for (auto i = 100; i < 199; ++i) {
1463 ASSERT_OK(batch.Put(handles_[2], Key(i), rnd.RandomString(1024)));
1464 }
1465
1466 SyncPoint::GetInstance()->SetCallBack(
1467 "WritableFileWriter::Append:BeforePrepareWrite", [&](void*) {
1468 write_error++;
1469 if (write_error > 2) {
1470 fault_fs_->SetFilesystemActive(false,
1471 IOStatus::NoSpace("Out of space"));
1472 }
1473 });
1474 SyncPoint::GetInstance()->EnableProcessing();
1475 WriteOptions wopts;
1476 wopts.sync = true;
1e59de90
TL		 1477 Status s = dbfull()->Write(wopts, &batch);
1478 ASSERT_TRUE(s.IsNoSpace());
20effc67
TL		 1479 }
1480 SyncPoint::GetInstance()->DisableProcessing();
1e59de90
TL		 1481 // `ClearAllCallBacks()` is needed in addition to `DisableProcessing()` to
1482 // drain all callbacks. Otherwise, a pending callback in the background
1483 // could re-disable `fault_fs_` after we enable it below.
1484 SyncPoint::GetInstance()->ClearAllCallBacks();
20effc67
TL		 1485 fault_fs_->SetFilesystemActive(true);
1486 ASSERT_EQ(listener->WaitForRecovery(5000000), true);
1487
1488 for (auto i = 1; i < 4; ++i) {
1489 // Every CF should have been flushed
1490 ASSERT_EQ(NumTableFilesAtLevel(0, i), 1);
1491 }
1492
1493 for (auto i = 1; i < 4; ++i) {
1494 for (auto j = 0; j < 199; ++j) {
1495 if (j < 100) {
1496 ASSERT_NE(Get(i, Key(j)), "NOT_FOUND");
1497 } else {
1498 ASSERT_EQ(Get(i, Key(j)), "NOT_FOUND");
1499 }
1500 }
1501 }
1502 ReopenWithColumnFamilies({"default", "one", "two", "three"}, options);
1503 for (auto i = 1; i < 4; ++i) {
1504 for (auto j = 0; j < 199; ++j) {
1505 if (j < 100) {
1506 ASSERT_NE(Get(i, Key(j)), "NOT_FOUND");
1507 } else {
1508 ASSERT_EQ(Get(i, Key(j)), "NOT_FOUND");
1509 }
1510 }
1511 }
1512 Close();
1513}
1514
1515TEST_F(DBErrorHandlingFSTest, MultiDBCompactionError) {
1516 if (mem_env_ != nullptr) {
1517 ROCKSDB_GTEST_SKIP("Test requires non-mock environment");
1518 return;
1519 }
1520 FaultInjectionTestEnv* def_env = new FaultInjectionTestEnv(env_);
1521 std::vector<std::unique_ptr<Env>> fault_envs;
1522 std::vector<FaultInjectionTestFS*> fault_fs;
1523 std::vector<Options> options;
1524 std::vector<std::shared_ptr<ErrorHandlerFSListener>> listener;
1525 std::vector<DB*> db;
1526 std::shared_ptr<SstFileManager> sfm(NewSstFileManager(def_env));
1527 int kNumDbInstances = 3;
1528 Random rnd(301);
1529
1530 for (auto i = 0; i < kNumDbInstances; ++i) {
1531 listener.emplace_back(new ErrorHandlerFSListener());
1532 options.emplace_back(GetDefaultOptions());
1533 fault_fs.emplace_back(new FaultInjectionTestFS(env_->GetFileSystem()));
1534 std::shared_ptr<FileSystem> fs(fault_fs.back());
1535 fault_envs.emplace_back(new CompositeEnvWrapper(def_env, fs));
1536 options[i].env = fault_envs.back().get();
1537 options[i].create_if_missing = true;
1538 options[i].level0_file_num_compaction_trigger = 2;
1539 options[i].writable_file_max_buffer_size = 32768;
1540 options[i].listeners.emplace_back(listener[i]);
1541 options[i].sst_file_manager = sfm;
1542 DB* dbptr;
1543 char buf[16];
1544
1545 listener[i]->EnableAutoRecovery();
1546 // Setup for returning error for the 3rd SST, which would be level 1
1547 listener[i]->InjectFileCreationError(fault_fs[i], 3,
1548 IOStatus::NoSpace("Out of space"));
1549 snprintf(buf, sizeof(buf), "_%d", i);
1e59de90
TL		 1550 ASSERT_OK(DestroyDB(dbname_ + std::string(buf), options[i]));
1551 ASSERT_OK(DB::Open(options[i], dbname_ + std::string(buf), &dbptr));
20effc67
TL		 1552 db.emplace_back(dbptr);
1553 }
1554
1555 for (auto i = 0; i < kNumDbInstances; ++i) {
1556 WriteBatch batch;
1557
1558 for (auto j = 0; j <= 100; ++j) {
1559 ASSERT_OK(batch.Put(Key(j), rnd.RandomString(1024)));
1560 }
1561
1562 WriteOptions wopts;
1563 wopts.sync = true;
1e59de90
TL		 1564 ASSERT_OK(db[i]->Write(wopts, &batch));
1565 ASSERT_OK(db[i]->Flush(FlushOptions()));
20effc67
TL		 1566 }
1567
1568 def_env->SetFilesystemActive(false, Status::NoSpace("Out of space"));
1569 for (auto i = 0; i < kNumDbInstances; ++i) {
1570 WriteBatch batch;
1571
1572 // Write to one CF
1573 for (auto j = 100; j < 199; ++j) {
1574 ASSERT_OK(batch.Put(Key(j), rnd.RandomString(1024)));
1575 }
1576
1577 WriteOptions wopts;
1578 wopts.sync = true;
1e59de90
TL		 1579 ASSERT_OK(db[i]->Write(wopts, &batch));
1580 ASSERT_OK(db[i]->Flush(FlushOptions()));
20effc67
TL		 1581 }
1582
1583 for (auto i = 0; i < kNumDbInstances; ++i) {
1584 Status s = static_cast<DBImpl*>(db[i])->TEST_WaitForCompact(true);
1585 ASSERT_EQ(s.severity(), Status::Severity::kSoftError);
1586 fault_fs[i]->SetFilesystemActive(true);
1587 }
1588
1589 def_env->SetFilesystemActive(true);
1590 for (auto i = 0; i < kNumDbInstances; ++i) {
1591 std::string prop;
1592 ASSERT_EQ(listener[i]->WaitForRecovery(5000000), true);
1e59de90 1593 ASSERT_OK(static_cast<DBImpl*>(db[i])->TEST_WaitForCompact(true));
20effc67 1594 EXPECT_TRUE(db[i]->GetProperty(
1e59de90 1595 "rocksdb.num-files-at-level" + std::to_string(0), &prop));
20effc67
TL		 1596 EXPECT_EQ(atoi(prop.c_str()), 0);
1597 EXPECT_TRUE(db[i]->GetProperty(
1e59de90 1598 "rocksdb.num-files-at-level" + std::to_string(1), &prop));
20effc67
TL		 1599 EXPECT_EQ(atoi(prop.c_str()), 1);
1600 }
1601
1e59de90
TL		 1602 SstFileManagerImpl* sfmImpl =
1603 static_cast_with_check<SstFileManagerImpl>(sfm.get());
1604 sfmImpl->Close();
1605
20effc67
TL		 1606 for (auto i = 0; i < kNumDbInstances; ++i) {
1607 char buf[16];
1608 snprintf(buf, sizeof(buf), "_%d", i);
1609 delete db[i];
1610 fault_fs[i]->SetFilesystemActive(true);
1611 if (getenv("KEEP_DB")) {
1612 printf("DB is still at %s%s\n", dbname_.c_str(), buf);
1613 } else {
1e59de90 1614 ASSERT_OK(DestroyDB(dbname_ + std::string(buf), options[i]));
20effc67
TL		 1615 }
1616 }
1617 options.clear();
1618 sfm.reset();
1619 delete def_env;
1620}
1621
1622TEST_F(DBErrorHandlingFSTest, MultiDBVariousErrors) {
1623 if (mem_env_ != nullptr) {
1624 ROCKSDB_GTEST_SKIP("Test requires non-mock environment");
1625 return;
1626 }
1627 FaultInjectionTestEnv* def_env = new FaultInjectionTestEnv(env_);
1628 std::vector<std::unique_ptr<Env>> fault_envs;
1629 std::vector<FaultInjectionTestFS*> fault_fs;
1630 std::vector<Options> options;
1631 std::vector<std::shared_ptr<ErrorHandlerFSListener>> listener;
1632 std::vector<DB*> db;
1633 std::shared_ptr<SstFileManager> sfm(NewSstFileManager(def_env));
1634 int kNumDbInstances = 3;
1635 Random rnd(301);
1636
1637 for (auto i = 0; i < kNumDbInstances; ++i) {
1638 listener.emplace_back(new ErrorHandlerFSListener());
1639 options.emplace_back(GetDefaultOptions());
1640 fault_fs.emplace_back(new FaultInjectionTestFS(env_->GetFileSystem()));
1641 std::shared_ptr<FileSystem> fs(fault_fs.back());
1642 fault_envs.emplace_back(new CompositeEnvWrapper(def_env, fs));
1643 options[i].env = fault_envs.back().get();
1644 options[i].create_if_missing = true;
1645 options[i].level0_file_num_compaction_trigger = 2;
1646 options[i].writable_file_max_buffer_size = 32768;
1647 options[i].listeners.emplace_back(listener[i]);
1648 options[i].sst_file_manager = sfm;
1649 DB* dbptr;
1650 char buf[16];
1651
1652 listener[i]->EnableAutoRecovery();
1653 switch (i) {
1654 case 0:
1655 // Setup for returning error for the 3rd SST, which would be level 1
1656 listener[i]->InjectFileCreationError(fault_fs[i], 3,
1657 IOStatus::NoSpace("Out of space"));
1658 break;
1659 case 1:
1660 // Setup for returning error after the 1st SST, which would result
1661 // in a hard error
1662 listener[i]->InjectFileCreationError(fault_fs[i], 2,
1663 IOStatus::NoSpace("Out of space"));
1664 break;
1665 default:
1666 break;
1667 }
1668 snprintf(buf, sizeof(buf), "_%d", i);
1e59de90
TL		 1669 ASSERT_OK(DestroyDB(dbname_ + std::string(buf), options[i]));
1670 ASSERT_OK(DB::Open(options[i], dbname_ + std::string(buf), &dbptr));
20effc67
TL		 1671 db.emplace_back(dbptr);
1672 }
1673
1674 for (auto i = 0; i < kNumDbInstances; ++i) {
1675 WriteBatch batch;
1676
1677 for (auto j = 0; j <= 100; ++j) {
1678 ASSERT_OK(batch.Put(Key(j), rnd.RandomString(1024)));
1679 }
1680
1681 WriteOptions wopts;
1682 wopts.sync = true;
1e59de90
TL		 1683 ASSERT_OK(db[i]->Write(wopts, &batch));
1684 ASSERT_OK(db[i]->Flush(FlushOptions()));
20effc67
TL		 1685 }
1686
1687 def_env->SetFilesystemActive(false, Status::NoSpace("Out of space"));
1688 for (auto i = 0; i < kNumDbInstances; ++i) {
1689 WriteBatch batch;
1690
1691 // Write to one CF
1692 for (auto j = 100; j < 199; ++j) {
1693 ASSERT_OK(batch.Put(Key(j), rnd.RandomString(1024)));
1694 }
1695
1696 WriteOptions wopts;
1697 wopts.sync = true;
1e59de90 1698 ASSERT_OK(db[i]->Write(wopts, &batch));
20effc67 1699 if (i != 1) {
1e59de90 1700 ASSERT_OK(db[i]->Flush(FlushOptions()));
20effc67 1701 } else {
1e59de90 1702 ASSERT_TRUE(db[i]->Flush(FlushOptions()).IsNoSpace());
20effc67
TL		 1703 }
1704 }
1705
1706 for (auto i = 0; i < kNumDbInstances; ++i) {
1707 Status s = static_cast<DBImpl*>(db[i])->TEST_WaitForCompact(true);
1708 switch (i) {
1709 case 0:
1710 ASSERT_EQ(s.severity(), Status::Severity::kSoftError);
1711 break;
1712 case 1:
1713 ASSERT_EQ(s.severity(), Status::Severity::kHardError);
1714 break;
1715 case 2:
1e59de90 1716 ASSERT_OK(s);
20effc67
TL		 1717 break;
1718 }
1719 fault_fs[i]->SetFilesystemActive(true);
1720 }
1721
1722 def_env->SetFilesystemActive(true);
1723 for (auto i = 0; i < kNumDbInstances; ++i) {
1724 std::string prop;
1725 if (i < 2) {
1726 ASSERT_EQ(listener[i]->WaitForRecovery(5000000), true);
1727 }
1728 if (i == 1) {
1e59de90 1729 ASSERT_OK(static_cast<DBImpl*>(db[i])->TEST_WaitForCompact(true));
20effc67
TL		 1730 }
1731 EXPECT_TRUE(db[i]->GetProperty(
1e59de90 1732 "rocksdb.num-files-at-level" + std::to_string(0), &prop));
20effc67
TL		 1733 EXPECT_EQ(atoi(prop.c_str()), 0);
1734 EXPECT_TRUE(db[i]->GetProperty(
1e59de90 1735 "rocksdb.num-files-at-level" + std::to_string(1), &prop));
20effc67
TL		 1736 EXPECT_EQ(atoi(prop.c_str()), 1);
1737 }
1738
1e59de90
TL		 1739 SstFileManagerImpl* sfmImpl =
1740 static_cast_with_check<SstFileManagerImpl>(sfm.get());
1741 sfmImpl->Close();
1742
20effc67
TL		 1743 for (auto i = 0; i < kNumDbInstances; ++i) {
1744 char buf[16];
1745 snprintf(buf, sizeof(buf), "_%d", i);
1746 fault_fs[i]->SetFilesystemActive(true);
1747 delete db[i];
1748 if (getenv("KEEP_DB")) {
1749 printf("DB is still at %s%s\n", dbname_.c_str(), buf);
1750 } else {
1e59de90 1751 EXPECT_OK(DestroyDB(dbname_ + std::string(buf), options[i]));
20effc67
TL		 1752 }
1753 }
1754 options.clear();
1755 delete def_env;
1756}
1757
1758// When Put the KV-pair, the write option is set to disable WAL.
1759// If retryable error happens in this condition, map the bg error
1760// to soft error and trigger auto resume. During auto resume, SwitchMemtable
1761// is disabled to avoid small SST tables. Write can still be applied before
1762// the bg error is cleaned unless the memtable is full.
1e59de90 1763TEST_F(DBErrorHandlingFSTest, FLushWritNoWALRetryableErrorAutoRecover1) {
20effc67
TL		 1764 // Activate the FS before the first resume
1765 std::shared_ptr<ErrorHandlerFSListener> listener(
1766 new ErrorHandlerFSListener());
1767 Options options = GetDefaultOptions();
1768 options.env = fault_env_.get();
1769 options.create_if_missing = true;
1770 options.listeners.emplace_back(listener);
1771 options.max_bgerror_resume_count = 2;
1772 options.bgerror_resume_retry_interval = 100000; // 0.1 second
1e59de90 1773 options.statistics = CreateDBStatistics();
20effc67
TL		 1774 Status s;
1775
1776 listener->EnableAutoRecovery(false);
1777 DestroyAndReopen(options);
1778
1779 IOStatus error_msg = IOStatus::IOError("Retryable IO Error");
1780 error_msg.SetRetryable(true);
1781
1782 WriteOptions wo = WriteOptions();
1783 wo.disableWAL = true;
1784 ASSERT_OK(Put(Key(1), "val1", wo));
1785 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency(
1786 {{"RecoverFromRetryableBGIOError:LoopOut",
1787 "FLushWritNoWALRetryableeErrorAutoRecover1:1"}});
1788 SyncPoint::GetInstance()->SetCallBack(
1789 "BuildTable:BeforeFinishBuildTable",
1790 [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); });
1791
1792 SyncPoint::GetInstance()->EnableProcessing();
1793 s = Flush();
1794 ASSERT_EQ("val1", Get(Key(1)));
1795 ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError);
1796 TEST_SYNC_POINT("FLushWritNoWALRetryableeErrorAutoRecover1:1");
1797 ASSERT_EQ("val1", Get(Key(1)));
1798 ASSERT_EQ("val1", Get(Key(1)));
1799 SyncPoint::GetInstance()->DisableProcessing();
1800 fault_fs_->SetFilesystemActive(true);
1e59de90
TL		 1801 ASSERT_EQ(3, options.statistics->getAndResetTickerCount(
1802 ERROR_HANDLER_BG_ERROR_COUNT));
1803 ASSERT_EQ(3, options.statistics->getAndResetTickerCount(
1804 ERROR_HANDLER_BG_IO_ERROR_COUNT));
1805 ASSERT_EQ(3, options.statistics->getAndResetTickerCount(
1806 ERROR_HANDLER_BG_RETRYABLE_IO_ERROR_COUNT));
1807 ASSERT_EQ(1, options.statistics->getAndResetTickerCount(
1808 ERROR_HANDLER_AUTORESUME_COUNT));
1809 ASSERT_LE(0, options.statistics->getAndResetTickerCount(
1810 ERROR_HANDLER_AUTORESUME_RETRY_TOTAL_COUNT));
1811 ASSERT_LE(0, options.statistics->getAndResetTickerCount(
1812 ERROR_HANDLER_AUTORESUME_SUCCESS_COUNT));
1813 HistogramData autoresume_retry;
1814 options.statistics->histogramData(ERROR_HANDLER_AUTORESUME_RETRY_COUNT,
1815 &autoresume_retry);
1816 ASSERT_GE(autoresume_retry.max, 0);
20effc67
TL		 1817 ASSERT_OK(Put(Key(2), "val2", wo));
1818 s = Flush();
1819 // Since auto resume fails, the bg error is not cleand, flush will
1820 // return the bg_error set before.
1821 ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError);
1822 ASSERT_EQ("val2", Get(Key(2)));
1823
1824 // call auto resume
1e59de90 1825 ASSERT_OK(dbfull()->Resume());
20effc67 1826 ASSERT_OK(Put(Key(3), "val3", wo));
20effc67 1827 // After resume is successful, the flush should be ok.
1e59de90 1828 ASSERT_OK(Flush());
20effc67
TL		 1829 ASSERT_EQ("val3", Get(Key(3)));
1830 Destroy(options);
1831}
1832
1e59de90 1833TEST_F(DBErrorHandlingFSTest, FLushWritNoWALRetryableErrorAutoRecover2) {
20effc67
TL		 1834 // Activate the FS before the first resume
1835 std::shared_ptr<ErrorHandlerFSListener> listener(
1836 new ErrorHandlerFSListener());
1837 Options options = GetDefaultOptions();
1838 options.env = fault_env_.get();
1839 options.create_if_missing = true;
1840 options.listeners.emplace_back(listener);
1841 options.max_bgerror_resume_count = 2;
1842 options.bgerror_resume_retry_interval = 100000; // 0.1 second
1e59de90 1843 options.statistics = CreateDBStatistics();
20effc67
TL		 1844 Status s;
1845
1846 listener->EnableAutoRecovery(false);
1847 DestroyAndReopen(options);
1848
1849 IOStatus error_msg = IOStatus::IOError("Retryable IO Error");
1850 error_msg.SetRetryable(true);
1851
1852 WriteOptions wo = WriteOptions();
1853 wo.disableWAL = true;
1854 ASSERT_OK(Put(Key(1), "val1", wo));
1855 SyncPoint::GetInstance()->SetCallBack(
1856 "BuildTable:BeforeFinishBuildTable",
1857 [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); });
1858
1859 SyncPoint::GetInstance()->EnableProcessing();
1860 s = Flush();
1861 ASSERT_EQ("val1", Get(Key(1)));
1e59de90
TL		 1862 ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError);
1863 SyncPoint::GetInstance()->DisableProcessing();
1864 fault_fs_->SetFilesystemActive(true);
1865 ASSERT_EQ(listener->WaitForRecovery(5000000), true);
20effc67 1866 ASSERT_EQ("val1", Get(Key(1)));
1e59de90
TL		 1867 ASSERT_EQ(1, options.statistics->getAndResetTickerCount(
1868 ERROR_HANDLER_BG_ERROR_COUNT));
1869 ASSERT_EQ(1, options.statistics->getAndResetTickerCount(
1870 ERROR_HANDLER_BG_IO_ERROR_COUNT));
1871 ASSERT_EQ(1, options.statistics->getAndResetTickerCount(
1872 ERROR_HANDLER_BG_RETRYABLE_IO_ERROR_COUNT));
1873 ASSERT_EQ(1, options.statistics->getAndResetTickerCount(
1874 ERROR_HANDLER_AUTORESUME_COUNT));
1875 ASSERT_LE(0, options.statistics->getAndResetTickerCount(
1876 ERROR_HANDLER_AUTORESUME_RETRY_TOTAL_COUNT));
1877 ASSERT_LE(0, options.statistics->getAndResetTickerCount(
1878 ERROR_HANDLER_AUTORESUME_SUCCESS_COUNT));
1879 HistogramData autoresume_retry;
1880 options.statistics->histogramData(ERROR_HANDLER_AUTORESUME_RETRY_COUNT,
1881 &autoresume_retry);
1882 ASSERT_GE(autoresume_retry.max, 0);
1883 ASSERT_OK(Put(Key(2), "val2", wo));
20effc67 1884 s = Flush();
1e59de90
TL		 1885 // Since auto resume is successful, the bg error is cleaned, flush will
1886 // be successful.
1887 ASSERT_OK(s);
20effc67 1888 ASSERT_EQ("val2", Get(Key(2)));
20effc67
TL		 1889 Destroy(options);
1890}
1891
1e59de90
TL		 1892// Auto resume fromt the flush retryable IO error. Activate the FS before the
1893// first resume. Resume is successful
1894TEST_F(DBErrorHandlingFSTest, FLushWritRetryableErrorAutoRecover1) {
1895 // Activate the FS before the first resume
20effc67
TL		 1896 std::shared_ptr<ErrorHandlerFSListener> listener(
1897 new ErrorHandlerFSListener());
1898 Options options = GetDefaultOptions();
1899 options.env = fault_env_.get();
1900 options.create_if_missing = true;
1901 options.listeners.emplace_back(listener);
1902 options.max_bgerror_resume_count = 2;
1e59de90 1903 options.bgerror_resume_retry_interval = 100000; // 0.1 second
20effc67
TL		 1904 Status s;
1905
1906 listener->EnableAutoRecovery(false);
1907 DestroyAndReopen(options);
1908
1909 IOStatus error_msg = IOStatus::IOError("Retryable IO Error");
1910 error_msg.SetRetryable(true);
20effc67
TL		 1911
1912 ASSERT_OK(Put(Key(1), "val1"));
20effc67
TL		 1913 SyncPoint::GetInstance()->SetCallBack(
1914 "BuildTable:BeforeFinishBuildTable",
1915 [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); });
20effc67
TL		 1916
1917 SyncPoint::GetInstance()->EnableProcessing();
1918 s = Flush();
1e59de90 1919 ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError);
20effc67
TL		 1920 SyncPoint::GetInstance()->DisableProcessing();
1921 fault_fs_->SetFilesystemActive(true);
1e59de90 1922 ASSERT_EQ(listener->WaitForRecovery(5000000), true);
20effc67 1923
1e59de90 1924 ASSERT_EQ("val1", Get(Key(1)));
20effc67
TL		 1925 Reopen(options);
1926 ASSERT_EQ("val1", Get(Key(1)));
1927 ASSERT_OK(Put(Key(2), "val2"));
1e59de90 1928 ASSERT_OK(Flush());
20effc67
TL		 1929 ASSERT_EQ("val2", Get(Key(2)));
1930
1931 Destroy(options);
1932}
1933
1e59de90
TL		 1934// Auto resume fromt the flush retryable IO error and set the retry limit count.
1935// Never activate the FS and auto resume should fail at the end
1936TEST_F(DBErrorHandlingFSTest, FLushWritRetryableErrorAutoRecover2) {
1937 // Fail all the resume and let user to resume
20effc67
TL		 1938 std::shared_ptr<ErrorHandlerFSListener> listener(
1939 new ErrorHandlerFSListener());
1940 Options options = GetDefaultOptions();
1941 options.env = fault_env_.get();
1942 options.create_if_missing = true;
1943 options.listeners.emplace_back(listener);
1944 options.max_bgerror_resume_count = 2;
1e59de90 1945 options.bgerror_resume_retry_interval = 100000; // 0.1 second
20effc67
TL		 1946 Status s;
1947
1948 listener->EnableAutoRecovery(false);
1949 DestroyAndReopen(options);
1950
1951 IOStatus error_msg = IOStatus::IOError("Retryable IO Error");
1952 error_msg.SetRetryable(true);
1953
1954 ASSERT_OK(Put(Key(1), "val1"));
1955 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency(
1e59de90
TL		 1956 {{"FLushWritRetryableeErrorAutoRecover2:0",
1957 "RecoverFromRetryableBGIOError:BeforeStart"},
1958 {"RecoverFromRetryableBGIOError:LoopOut",
1959 "FLushWritRetryableeErrorAutoRecover2:1"}});
20effc67
TL		 1960 SyncPoint::GetInstance()->SetCallBack(
1961 "BuildTable:BeforeFinishBuildTable",
1962 [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); });
1963 SyncPoint::GetInstance()->EnableProcessing();
1964 s = Flush();
1e59de90
TL		 1965 ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError);
1966 TEST_SYNC_POINT("FLushWritRetryableeErrorAutoRecover2:0");
1967 TEST_SYNC_POINT("FLushWritRetryableeErrorAutoRecover2:1");
1968 fault_fs_->SetFilesystemActive(true);
20effc67
TL		 1969 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
1970 SyncPoint::GetInstance()->DisableProcessing();
20effc67 1971
1e59de90
TL		 1972 ASSERT_EQ("val1", Get(Key(1)));
1973 // Auto resume fails due to FS does not recover during resume. User call
1974 // resume manually here.
1975 s = dbfull()->Resume();
1976 ASSERT_EQ("val1", Get(Key(1)));
1977 ASSERT_OK(s);
20effc67 1978 ASSERT_OK(Put(Key(2), "val2"));
1e59de90 1979 ASSERT_OK(Flush());
20effc67
TL		 1980 ASSERT_EQ("val2", Get(Key(2)));
1981
1982 Destroy(options);
1983}
1984
1e59de90
TL		 1985// Auto resume fromt the flush retryable IO error and set the retry limit count.
1986// Fail the first resume and let the second resume be successful.
1987TEST_F(DBErrorHandlingFSTest, ManifestWriteRetryableErrorAutoRecover) {
1988 // Fail the first resume and let the second resume be successful
20effc67
TL		 1989 std::shared_ptr<ErrorHandlerFSListener> listener(
1990 new ErrorHandlerFSListener());
1991 Options options = GetDefaultOptions();
1992 options.env = fault_env_.get();
1993 options.create_if_missing = true;
1994 options.listeners.emplace_back(listener);
1995 options.max_bgerror_resume_count = 2;
1e59de90 1996 options.bgerror_resume_retry_interval = 100000; // 0.1 second
20effc67 1997 Status s;
1e59de90
TL		 1998 std::string old_manifest;
1999 std::string new_manifest;
20effc67
TL		 2000
2001 listener->EnableAutoRecovery(false);
2002 DestroyAndReopen(options);
1e59de90 2003 old_manifest = GetManifestNameFromLiveFiles();
20effc67
TL		 2004
2005 IOStatus error_msg = IOStatus::IOError("Retryable IO Error");
2006 error_msg.SetRetryable(true);
2007
1e59de90
TL		 2008 ASSERT_OK(Put(Key(0), "val"));
2009 ASSERT_OK(Flush());
2010 ASSERT_OK(Put(Key(1), "val"));
20effc67 2011 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency(
1e59de90
TL		 2012 {{"RecoverFromRetryableBGIOError:BeforeStart",
2013 "ManifestWriteRetryableErrorAutoRecover:0"},
2014 {"ManifestWriteRetryableErrorAutoRecover:1",
20effc67 2015 "RecoverFromRetryableBGIOError:BeforeWait1"},
1e59de90
TL		 2016 {"RecoverFromRetryableBGIOError:RecoverSuccess",
2017 "ManifestWriteRetryableErrorAutoRecover:2"}});
20effc67 2018 SyncPoint::GetInstance()->SetCallBack(
1e59de90 2019 "VersionSet::LogAndApply:WriteManifest",
20effc67
TL		 2020 [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); });
2021 SyncPoint::GetInstance()->EnableProcessing();
2022 s = Flush();
1e59de90
TL		 2023 ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError);
2024 TEST_SYNC_POINT("ManifestWriteRetryableErrorAutoRecover:0");
20effc67
TL		 2025 fault_fs_->SetFilesystemActive(true);
2026 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
1e59de90
TL		 2027 TEST_SYNC_POINT("ManifestWriteRetryableErrorAutoRecover:1");
2028 TEST_SYNC_POINT("ManifestWriteRetryableErrorAutoRecover:2");
20effc67
TL		 2029 SyncPoint::GetInstance()->DisableProcessing();
2030
1e59de90
TL		 2031 new_manifest = GetManifestNameFromLiveFiles();
2032 ASSERT_NE(new_manifest, old_manifest);
20effc67 2033
1e59de90
TL		 2034 Reopen(options);
2035 ASSERT_EQ("val", Get(Key(0)));
2036 ASSERT_EQ("val", Get(Key(1)));
2037 Close();
20effc67
TL		 2038}
2039
1e59de90 2040TEST_F(DBErrorHandlingFSTest, ManifestWriteNoWALRetryableErrorAutoRecover) {
20effc67
TL		 2041 // Fail the first resume and let the second resume be successful
2042 std::shared_ptr<ErrorHandlerFSListener> listener(
2043 new ErrorHandlerFSListener());
2044 Options options = GetDefaultOptions();
2045 options.env = fault_env_.get();
2046 options.create_if_missing = true;
2047 options.listeners.emplace_back(listener);
2048 options.max_bgerror_resume_count = 2;
2049 options.bgerror_resume_retry_interval = 100000; // 0.1 second
2050 Status s;
2051 std::string old_manifest;
2052 std::string new_manifest;
2053
2054 listener->EnableAutoRecovery(false);
2055 DestroyAndReopen(options);
2056 old_manifest = GetManifestNameFromLiveFiles();
2057
2058 IOStatus error_msg = IOStatus::IOError("Retryable IO Error");
2059 error_msg.SetRetryable(true);
2060
1e59de90
TL		 2061 WriteOptions wo = WriteOptions();
2062 wo.disableWAL = true;
2063 ASSERT_OK(Put(Key(0), "val", wo));
20effc67 2064 ASSERT_OK(Flush());
1e59de90 2065 ASSERT_OK(Put(Key(1), "val", wo));
20effc67
TL		 2066 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency(
2067 {{"RecoverFromRetryableBGIOError:BeforeStart",
1e59de90
TL		 2068 "ManifestWriteNoWALRetryableErrorAutoRecover:0"},
2069 {"ManifestWriteNoWALRetryableErrorAutoRecover:1",
20effc67
TL		 2070 "RecoverFromRetryableBGIOError:BeforeWait1"},
2071 {"RecoverFromRetryableBGIOError:RecoverSuccess",
1e59de90 2072 "ManifestWriteNoWALRetryableErrorAutoRecover:2"}});
20effc67
TL		 2073 SyncPoint::GetInstance()->SetCallBack(
2074 "VersionSet::LogAndApply:WriteManifest",
2075 [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); });
2076 SyncPoint::GetInstance()->EnableProcessing();
2077 s = Flush();
1e59de90
TL		 2078 ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError);
2079 TEST_SYNC_POINT("ManifestWriteNoWALRetryableErrorAutoRecover:0");
20effc67
TL		 2080 fault_fs_->SetFilesystemActive(true);
2081 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
1e59de90
TL		 2082 TEST_SYNC_POINT("ManifestWriteNoWALRetryableErrorAutoRecover:1");
2083 TEST_SYNC_POINT("ManifestWriteNoWALRetryableErrorAutoRecover:2");
20effc67
TL		 2084 SyncPoint::GetInstance()->DisableProcessing();
2085
2086 new_manifest = GetManifestNameFromLiveFiles();
2087 ASSERT_NE(new_manifest, old_manifest);
2088
2089 Reopen(options);
2090 ASSERT_EQ("val", Get(Key(0)));
2091 ASSERT_EQ("val", Get(Key(1)));
2092 Close();
2093}
2094
2095TEST_F(DBErrorHandlingFSTest,
2096 CompactionManifestWriteRetryableErrorAutoRecover) {
2097 std::shared_ptr<ErrorHandlerFSListener> listener(
2098 new ErrorHandlerFSListener());
2099 Options options = GetDefaultOptions();
2100 options.env = fault_env_.get();
2101 options.create_if_missing = true;
2102 options.level0_file_num_compaction_trigger = 2;
2103 options.listeners.emplace_back(listener);
2104 options.max_bgerror_resume_count = 2;
2105 options.bgerror_resume_retry_interval = 100000; // 0.1 second
2106 Status s;
2107 std::string old_manifest;
2108 std::string new_manifest;
2109 std::atomic<bool> fail_manifest(false);
2110 DestroyAndReopen(options);
2111 old_manifest = GetManifestNameFromLiveFiles();
2112
2113 IOStatus error_msg = IOStatus::IOError("Retryable IO Error");
2114 error_msg.SetRetryable(true);
2115
2116 ASSERT_OK(Put(Key(0), "val"));
2117 ASSERT_OK(Put(Key(2), "val"));
1e59de90 2118 ASSERT_OK(Flush());
20effc67
TL		 2119
2120 listener->OverrideBGError(Status(error_msg, Status::Severity::kHardError));
2121 listener->EnableAutoRecovery(false);
2122 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency(
2123 // Wait for flush of 2nd L0 file before starting compaction
2124 {{"DBImpl::FlushMemTable:FlushMemTableFinished",
2125 "BackgroundCallCompaction:0"},
2126 // Wait for compaction to detect manifest write error
2127 {"BackgroundCallCompaction:1", "CompactionManifestWriteErrorAR:0"},
2128 // Make compaction thread wait for error to be cleared
2129 {"CompactionManifestWriteErrorAR:1",
2130 "DBImpl::BackgroundCallCompaction:FoundObsoleteFiles"},
2131 {"CompactionManifestWriteErrorAR:2",
2132 "RecoverFromRetryableBGIOError:BeforeStart"},
2133 // Fail the first resume, before the wait in resume
2134 {"RecoverFromRetryableBGIOError:BeforeResume0",
2135 "CompactionManifestWriteErrorAR:3"},
2136 // Activate the FS before the second resume
2137 {"CompactionManifestWriteErrorAR:4",
2138 "RecoverFromRetryableBGIOError:BeforeResume1"},
2139 // Wait the auto resume be sucessful
2140 {"RecoverFromRetryableBGIOError:RecoverSuccess",
2141 "CompactionManifestWriteErrorAR:5"}});
2142 // trigger manifest write failure in compaction thread
2143 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
2144 "BackgroundCallCompaction:0", [&](void*) { fail_manifest.store(true); });
2145 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
2146 "VersionSet::LogAndApply:WriteManifest", [&](void*) {
2147 if (fail_manifest.load()) {
2148 fault_fs_->SetFilesystemActive(false, error_msg);
2149 }
2150 });
2151 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
2152
2153 ASSERT_OK(Put(Key(1), "val"));
2154 s = Flush();
1e59de90 2155 ASSERT_OK(s);
20effc67
TL		 2156
2157 TEST_SYNC_POINT("CompactionManifestWriteErrorAR:0");
2158 TEST_SYNC_POINT("CompactionManifestWriteErrorAR:1");
2159
2160 s = dbfull()->TEST_WaitForCompact();
2161 ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError);
2162 TEST_SYNC_POINT("CompactionManifestWriteErrorAR:2");
2163 TEST_SYNC_POINT("CompactionManifestWriteErrorAR:3");
2164 fault_fs_->SetFilesystemActive(true);
2165 SyncPoint::GetInstance()->ClearAllCallBacks();
2166 TEST_SYNC_POINT("CompactionManifestWriteErrorAR:4");
2167 TEST_SYNC_POINT("CompactionManifestWriteErrorAR:5");
2168 SyncPoint::GetInstance()->DisableProcessing();
2169
2170 new_manifest = GetManifestNameFromLiveFiles();
2171 ASSERT_NE(new_manifest, old_manifest);
2172
2173 Reopen(options);
2174 ASSERT_EQ("val", Get(Key(0)));
2175 ASSERT_EQ("val", Get(Key(1)));
2176 ASSERT_EQ("val", Get(Key(2)));
2177 Close();
2178}
2179
2180TEST_F(DBErrorHandlingFSTest, CompactionWriteRetryableErrorAutoRecover) {
2181 // In this test, in the first round of compaction, the FS is set to error.
2182 // So the first compaction fails due to retryable IO error and it is mapped
2183 // to soft error. Then, compaction is rescheduled, in the second round of
2184 // compaction, the FS is set to active and compaction is successful, so
2185 // the test will hit the CompactionJob::FinishCompactionOutputFile1 sync
2186 // point.
2187 std::shared_ptr<ErrorHandlerFSListener> listener(
2188 new ErrorHandlerFSListener());
2189 Options options = GetDefaultOptions();
2190 options.env = fault_env_.get();
2191 options.create_if_missing = true;
2192 options.level0_file_num_compaction_trigger = 2;
2193 options.listeners.emplace_back(listener);
2194 Status s;
2195 std::atomic<bool> fail_first(false);
2196 std::atomic<bool> fail_second(true);
2197 DestroyAndReopen(options);
2198
2199 IOStatus error_msg = IOStatus::IOError("Retryable IO Error");
2200 error_msg.SetRetryable(true);
2201
2202 ASSERT_OK(Put(Key(0), "va;"));
2203 ASSERT_OK(Put(Key(2), "va;"));
2204 s = Flush();
1e59de90 2205 ASSERT_OK(s);
20effc67
TL		 2206
2207 listener->OverrideBGError(Status(error_msg, Status::Severity::kHardError));
2208 listener->EnableAutoRecovery(false);
2209 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency(
2210 {{"DBImpl::FlushMemTable:FlushMemTableFinished",
2211 "BackgroundCallCompaction:0"},
2212 {"CompactionJob::FinishCompactionOutputFile1",
2213 "CompactionWriteRetryableErrorAutoRecover0"}});
2214 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
2215 "DBImpl::BackgroundCompaction:Start",
2216 [&](void*) { fault_fs_->SetFilesystemActive(true); });
2217 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
2218 "BackgroundCallCompaction:0", [&](void*) { fail_first.store(true); });
2219 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
2220 "CompactionJob::OpenCompactionOutputFile", [&](void*) {
2221 if (fail_first.load() && fail_second.load()) {
2222 fault_fs_->SetFilesystemActive(false, error_msg);
2223 fail_second.store(false);
2224 }
2225 });
2226 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
2227
2228 ASSERT_OK(Put(Key(1), "val"));
2229 s = Flush();
1e59de90 2230 ASSERT_OK(s);
20effc67
TL		 2231
2232 s = dbfull()->TEST_WaitForCompact();
1e59de90 2233 ASSERT_OK(s);
20effc67
TL		 2234 TEST_SYNC_POINT("CompactionWriteRetryableErrorAutoRecover0");
2235 SyncPoint::GetInstance()->ClearAllCallBacks();
2236 SyncPoint::GetInstance()->DisableProcessing();
2237 Destroy(options);
2238}
2239
2240TEST_F(DBErrorHandlingFSTest, WALWriteRetryableErrorAutoRecover1) {
2241 std::shared_ptr<ErrorHandlerFSListener> listener(
2242 new ErrorHandlerFSListener());
2243 Options options = GetDefaultOptions();
2244 options.env = fault_env_.get();
2245 options.create_if_missing = true;
2246 options.writable_file_max_buffer_size = 32768;
2247 options.listeners.emplace_back(listener);
2248 options.paranoid_checks = true;
2249 options.max_bgerror_resume_count = 2;
2250 options.bgerror_resume_retry_interval = 100000; // 0.1 second
2251 Status s;
2252 Random rnd(301);
2253
2254 DestroyAndReopen(options);
2255
2256 IOStatus error_msg = IOStatus::IOError("Retryable IO Error");
2257 error_msg.SetRetryable(true);
2258
2259 // For the first batch, write is successful, require sync
2260 {
2261 WriteBatch batch;
2262
2263 for (auto i = 0; i < 100; ++i) {
2264 ASSERT_OK(batch.Put(Key(i), rnd.RandomString(1024)));
2265 }
2266
2267 WriteOptions wopts;
2268 wopts.sync = true;
1e59de90 2269 ASSERT_OK(dbfull()->Write(wopts, &batch));
20effc67
TL		 2270 };
2271
2272 // For the second batch, the first 2 file Append are successful, then the
2273 // following Append fails due to file system retryable IOError.
2274 {
2275 WriteBatch batch;
2276 int write_error = 0;
2277
2278 for (auto i = 100; i < 200; ++i) {
2279 ASSERT_OK(batch.Put(Key(i), rnd.RandomString(1024)));
2280 }
2281 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency(
1e59de90
TL		 2282 {{"WALWriteErrorDone", "RecoverFromRetryableBGIOError:BeforeStart"},
2283 {"RecoverFromRetryableBGIOError:BeforeResume0", "WALWriteError1:0"},
20effc67
TL		 2284 {"WALWriteError1:1", "RecoverFromRetryableBGIOError:BeforeResume1"},
2285 {"RecoverFromRetryableBGIOError:RecoverSuccess", "WALWriteError1:2"}});
2286
2287 SyncPoint::GetInstance()->SetCallBack(
2288 "WritableFileWriter::Append:BeforePrepareWrite", [&](void*) {
2289 write_error++;
2290 if (write_error > 2) {
2291 fault_fs_->SetFilesystemActive(false, error_msg);
2292 }
2293 });
2294 SyncPoint::GetInstance()->EnableProcessing();
2295 WriteOptions wopts;
2296 wopts.sync = true;
2297 s = dbfull()->Write(wopts, &batch);
2298 ASSERT_EQ(true, s.IsIOError());
1e59de90 2299 TEST_SYNC_POINT("WALWriteErrorDone");
20effc67
TL		 2300
2301 TEST_SYNC_POINT("WALWriteError1:0");
2302 fault_fs_->SetFilesystemActive(true);
2303 SyncPoint::GetInstance()->ClearAllCallBacks();
2304 TEST_SYNC_POINT("WALWriteError1:1");
2305 TEST_SYNC_POINT("WALWriteError1:2");
2306 }
2307 SyncPoint::GetInstance()->DisableProcessing();
2308
2309 // Data in corrupted WAL are not stored
2310 for (auto i = 0; i < 199; ++i) {
2311 if (i < 100) {
2312 ASSERT_NE(Get(Key(i)), "NOT_FOUND");
2313 } else {
2314 ASSERT_EQ(Get(Key(i)), "NOT_FOUND");
2315 }
2316 }
2317
2318 // Resume and write a new batch, should be in the WAL
2319 {
2320 WriteBatch batch;
2321
2322 for (auto i = 200; i < 300; ++i) {
2323 ASSERT_OK(batch.Put(Key(i), rnd.RandomString(1024)));
2324 }
2325
2326 WriteOptions wopts;
2327 wopts.sync = true;
1e59de90 2328 ASSERT_OK(dbfull()->Write(wopts, &batch));
20effc67
TL		 2329 };
2330
2331 Reopen(options);
2332 for (auto i = 0; i < 300; ++i) {
2333 if (i < 100 || i >= 200) {
2334 ASSERT_NE(Get(Key(i)), "NOT_FOUND");
2335 } else {
2336 ASSERT_EQ(Get(Key(i)), "NOT_FOUND");
2337 }
2338 }
2339 Close();
2340}
2341
2342TEST_F(DBErrorHandlingFSTest, WALWriteRetryableErrorAutoRecover2) {
2343 // Fail the first recover and try second time.
2344 std::shared_ptr<ErrorHandlerFSListener> listener(
2345 new ErrorHandlerFSListener());
2346 Options options = GetDefaultOptions();
2347 options.env = fault_env_.get();
2348 options.create_if_missing = true;
2349 options.writable_file_max_buffer_size = 32768;
2350 options.listeners.emplace_back(listener);
2351 options.paranoid_checks = true;
2352 options.max_bgerror_resume_count = 2;
2353 options.bgerror_resume_retry_interval = 100000; // 0.1 second
2354 Status s;
2355 Random rnd(301);
2356
2357 DestroyAndReopen(options);
2358
2359 IOStatus error_msg = IOStatus::IOError("Retryable IO Error");
2360 error_msg.SetRetryable(true);
2361
2362 // For the first batch, write is successful, require sync
2363 {
2364 WriteBatch batch;
2365
2366 for (auto i = 0; i < 100; ++i) {
2367 ASSERT_OK(batch.Put(Key(i), rnd.RandomString(1024)));
2368 }
2369
2370 WriteOptions wopts;
2371 wopts.sync = true;
1e59de90 2372 ASSERT_OK(dbfull()->Write(wopts, &batch));
20effc67
TL		 2373 };
2374
2375 // For the second batch, the first 2 file Append are successful, then the
2376 // following Append fails due to file system retryable IOError.
2377 {
2378 WriteBatch batch;
2379 int write_error = 0;
2380
2381 for (auto i = 100; i < 200; ++i) {
2382 ASSERT_OK(batch.Put(Key(i), rnd.RandomString(1024)));
2383 }
2384 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency(
2385 {{"RecoverFromRetryableBGIOError:BeforeWait0", "WALWriteError2:0"},
2386 {"WALWriteError2:1", "RecoverFromRetryableBGIOError:BeforeWait1"},
2387 {"RecoverFromRetryableBGIOError:RecoverSuccess", "WALWriteError2:2"}});
2388
2389 SyncPoint::GetInstance()->SetCallBack(
2390 "WritableFileWriter::Append:BeforePrepareWrite", [&](void*) {
2391 write_error++;
2392 if (write_error > 2) {
2393 fault_fs_->SetFilesystemActive(false, error_msg);
2394 }
2395 });
2396 SyncPoint::GetInstance()->EnableProcessing();
2397 WriteOptions wopts;
2398 wopts.sync = true;
2399 s = dbfull()->Write(wopts, &batch);
2400 ASSERT_EQ(true, s.IsIOError());
2401
2402 TEST_SYNC_POINT("WALWriteError2:0");
2403 fault_fs_->SetFilesystemActive(true);
2404 SyncPoint::GetInstance()->ClearAllCallBacks();
2405 TEST_SYNC_POINT("WALWriteError2:1");
2406 TEST_SYNC_POINT("WALWriteError2:2");
2407 }
2408 SyncPoint::GetInstance()->DisableProcessing();
2409
2410 // Data in corrupted WAL are not stored
2411 for (auto i = 0; i < 199; ++i) {
2412 if (i < 100) {
2413 ASSERT_NE(Get(Key(i)), "NOT_FOUND");
2414 } else {
2415 ASSERT_EQ(Get(Key(i)), "NOT_FOUND");
2416 }
2417 }
2418
2419 // Resume and write a new batch, should be in the WAL
2420 {
2421 WriteBatch batch;
2422
2423 for (auto i = 200; i < 300; ++i) {
2424 ASSERT_OK(batch.Put(Key(i), rnd.RandomString(1024)));
2425 }
2426
2427 WriteOptions wopts;
2428 wopts.sync = true;
1e59de90 2429 ASSERT_OK(dbfull()->Write(wopts, &batch));
20effc67
TL		 2430 };
2431
2432 Reopen(options);
2433 for (auto i = 0; i < 300; ++i) {
2434 if (i < 100 || i >= 200) {
2435 ASSERT_NE(Get(Key(i)), "NOT_FOUND");
2436 } else {
2437 ASSERT_EQ(Get(Key(i)), "NOT_FOUND");
2438 }
2439 }
2440 Close();
2441}
2442
1e59de90
TL		 2443// Fail auto resume from a flush retryable error and verify that
2444// OnErrorRecoveryEnd listener callback is called
2445TEST_F(DBErrorHandlingFSTest, FLushWritRetryableErrorAbortRecovery) {
2446 // Activate the FS before the first resume
2447 std::shared_ptr<ErrorHandlerFSListener> listener(
2448 new ErrorHandlerFSListener());
2449 Options options = GetDefaultOptions();
2450 options.env = fault_env_.get();
2451 options.create_if_missing = true;
2452 options.listeners.emplace_back(listener);
2453 options.max_bgerror_resume_count = 2;
2454 options.bgerror_resume_retry_interval = 100000; // 0.1 second
2455 Status s;
2456
2457 listener->EnableAutoRecovery(false);
2458 DestroyAndReopen(options);
2459
2460 IOStatus error_msg = IOStatus::IOError("Retryable IO Error");
2461 error_msg.SetRetryable(true);
2462
2463 ASSERT_OK(Put(Key(1), "val1"));
2464 SyncPoint::GetInstance()->SetCallBack(
2465 "BuildTable:BeforeFinishBuildTable",
2466 [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); });
2467
2468 SyncPoint::GetInstance()->EnableProcessing();
2469 s = Flush();
2470 ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError);
2471 ASSERT_EQ(listener->WaitForRecovery(5000000), true);
2472 ASSERT_EQ(listener->new_bg_error(), Status::Aborted());
2473 SyncPoint::GetInstance()->DisableProcessing();
2474 fault_fs_->SetFilesystemActive(true);
2475
2476 Destroy(options);
2477}
2478
2479TEST_F(DBErrorHandlingFSTest, FlushReadError) {
2480 std::shared_ptr<ErrorHandlerFSListener> listener =
2481 std::make_shared<ErrorHandlerFSListener>();
2482 Options options = GetDefaultOptions();
2483 options.env = fault_env_.get();
2484 options.create_if_missing = true;
2485 options.listeners.emplace_back(listener);
2486 options.statistics = CreateDBStatistics();
2487 Status s;
2488
2489 listener->EnableAutoRecovery(false);
2490 DestroyAndReopen(options);
2491
2492 ASSERT_OK(Put(Key(0), "val"));
2493 SyncPoint::GetInstance()->SetCallBack(
2494 "BuildTable:BeforeOutputValidation", [&](void*) {
2495 IOStatus st = IOStatus::IOError();
2496 st.SetRetryable(true);
2497 st.SetScope(IOStatus::IOErrorScope::kIOErrorScopeFile);
2498 fault_fs_->SetFilesystemActive(false, st);
2499 });
2500 SyncPoint::GetInstance()->SetCallBack(
2501 "BuildTable:BeforeDeleteFile",
2502 [&](void*) { fault_fs_->SetFilesystemActive(true, IOStatus::OK()); });
2503 SyncPoint::GetInstance()->EnableProcessing();
2504 s = Flush();
2505 ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError);
2506 SyncPoint::GetInstance()->DisableProcessing();
2507 fault_fs_->SetFilesystemActive(true);
2508 ASSERT_EQ(listener->WaitForRecovery(5000000), true);
2509 ASSERT_EQ(1, options.statistics->getAndResetTickerCount(
2510 ERROR_HANDLER_BG_ERROR_COUNT));
2511 ASSERT_EQ(1, options.statistics->getAndResetTickerCount(
2512 ERROR_HANDLER_BG_IO_ERROR_COUNT));
2513 ASSERT_EQ(1, options.statistics->getAndResetTickerCount(
2514 ERROR_HANDLER_BG_RETRYABLE_IO_ERROR_COUNT));
2515 ASSERT_LE(1, options.statistics->getAndResetTickerCount(
2516 ERROR_HANDLER_AUTORESUME_COUNT));
2517 ASSERT_LE(0, options.statistics->getAndResetTickerCount(
2518 ERROR_HANDLER_AUTORESUME_RETRY_TOTAL_COUNT));
2519 s = dbfull()->TEST_GetBGError();
2520 ASSERT_OK(s);
2521
2522 Reopen(GetDefaultOptions());
2523 ASSERT_EQ("val", Get(Key(0)));
2524}
2525
2526TEST_F(DBErrorHandlingFSTest, AtomicFlushReadError) {
2527 std::shared_ptr<ErrorHandlerFSListener> listener =
2528 std::make_shared<ErrorHandlerFSListener>();
2529 Options options = GetDefaultOptions();
2530 options.env = fault_env_.get();
2531 options.create_if_missing = true;
2532 options.listeners.emplace_back(listener);
2533 options.statistics = CreateDBStatistics();
2534 Status s;
2535
2536 listener->EnableAutoRecovery(false);
2537 options.atomic_flush = true;
2538 CreateAndReopenWithCF({"pikachu"}, options);
2539
2540 ASSERT_OK(Put(0, Key(0), "val"));
2541 ASSERT_OK(Put(1, Key(0), "val"));
2542 SyncPoint::GetInstance()->SetCallBack(
2543 "BuildTable:BeforeOutputValidation", [&](void*) {
2544 IOStatus st = IOStatus::IOError();
2545 st.SetRetryable(true);
2546 st.SetScope(IOStatus::IOErrorScope::kIOErrorScopeFile);
2547 fault_fs_->SetFilesystemActive(false, st);
2548 });
2549 SyncPoint::GetInstance()->SetCallBack(
2550 "BuildTable:BeforeDeleteFile",
2551 [&](void*) { fault_fs_->SetFilesystemActive(true, IOStatus::OK()); });
2552 SyncPoint::GetInstance()->EnableProcessing();
2553 s = Flush({0, 1});
2554 ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError);
2555 SyncPoint::GetInstance()->DisableProcessing();
2556 fault_fs_->SetFilesystemActive(true);
2557 ASSERT_EQ(listener->WaitForRecovery(5000000), true);
2558 ASSERT_EQ(1, options.statistics->getAndResetTickerCount(
2559 ERROR_HANDLER_BG_ERROR_COUNT));
2560 ASSERT_EQ(1, options.statistics->getAndResetTickerCount(
2561 ERROR_HANDLER_BG_IO_ERROR_COUNT));
2562 ASSERT_EQ(1, options.statistics->getAndResetTickerCount(
2563 ERROR_HANDLER_BG_RETRYABLE_IO_ERROR_COUNT));
2564 ASSERT_LE(1, options.statistics->getAndResetTickerCount(
2565 ERROR_HANDLER_AUTORESUME_COUNT));
2566 ASSERT_LE(0, options.statistics->getAndResetTickerCount(
2567 ERROR_HANDLER_AUTORESUME_RETRY_TOTAL_COUNT));
2568 s = dbfull()->TEST_GetBGError();
2569 ASSERT_OK(s);
2570
2571 TryReopenWithColumnFamilies({kDefaultColumnFamilyName, "pikachu"},
2572 GetDefaultOptions());
2573 ASSERT_EQ("val", Get(Key(0)));
2574}
2575
2576TEST_F(DBErrorHandlingFSTest, AtomicFlushNoSpaceError) {
2577 std::shared_ptr<ErrorHandlerFSListener> listener =
2578 std::make_shared<ErrorHandlerFSListener>();
2579 Options options = GetDefaultOptions();
2580 options.env = fault_env_.get();
2581 options.create_if_missing = true;
2582 options.listeners.emplace_back(listener);
2583 options.statistics = CreateDBStatistics();
2584 Status s;
2585
2586 listener->EnableAutoRecovery(true);
2587 options.atomic_flush = true;
2588 CreateAndReopenWithCF({"pikachu"}, options);
2589
2590 ASSERT_OK(Put(0, Key(0), "val"));
2591 ASSERT_OK(Put(1, Key(0), "val"));
2592 SyncPoint::GetInstance()->SetCallBack("BuildTable:create_file", [&](void*) {
2593 IOStatus st = IOStatus::NoSpace();
2594 fault_fs_->SetFilesystemActive(false, st);
2595 });
2596 SyncPoint::GetInstance()->SetCallBack(
2597 "BuildTable:BeforeDeleteFile",
2598 [&](void*) { fault_fs_->SetFilesystemActive(true, IOStatus::OK()); });
2599 SyncPoint::GetInstance()->EnableProcessing();
2600 s = Flush({0, 1});
2601 ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError);
2602 SyncPoint::GetInstance()->DisableProcessing();
2603 fault_fs_->SetFilesystemActive(true);
2604 ASSERT_EQ(listener->WaitForRecovery(5000000), true);
2605 ASSERT_LE(1, options.statistics->getAndResetTickerCount(
2606 ERROR_HANDLER_BG_ERROR_COUNT));
2607 ASSERT_LE(1, options.statistics->getAndResetTickerCount(
2608 ERROR_HANDLER_BG_IO_ERROR_COUNT));
2609 s = dbfull()->TEST_GetBGError();
2610 ASSERT_OK(s);
2611
2612 TryReopenWithColumnFamilies({kDefaultColumnFamilyName, "pikachu"},
2613 GetDefaultOptions());
2614 ASSERT_EQ("val", Get(Key(0)));
2615}
2616
2617TEST_F(DBErrorHandlingFSTest, CompactionReadRetryableErrorAutoRecover) {
2618 // In this test, in the first round of compaction, the FS is set to error.
2619 // So the first compaction fails due to retryable IO error and it is mapped
2620 // to soft error. Then, compaction is rescheduled, in the second round of
2621 // compaction, the FS is set to active and compaction is successful, so
2622 // the test will hit the CompactionJob::FinishCompactionOutputFile1 sync
2623 // point.
2624 std::shared_ptr<ErrorHandlerFSListener> listener =
2625 std::make_shared<ErrorHandlerFSListener>();
2626 Options options = GetDefaultOptions();
2627 options.env = fault_env_.get();
2628 options.create_if_missing = true;
2629 options.level0_file_num_compaction_trigger = 2;
2630 options.listeners.emplace_back(listener);
2631 BlockBasedTableOptions table_options;
2632 table_options.no_block_cache = true;
2633 options.table_factory.reset(NewBlockBasedTableFactory(table_options));
2634 Status s;
2635 std::atomic<bool> fail_first(false);
2636 std::atomic<bool> fail_second(true);
2637 Random rnd(301);
2638 DestroyAndReopen(options);
2639
2640 IOStatus error_msg = IOStatus::IOError("Retryable IO Error");
2641 error_msg.SetRetryable(true);
2642
2643 for (int i = 0; i < 100; ++i) {
2644 ASSERT_OK(Put(Key(i), rnd.RandomString(1024)));
2645 }
2646 s = Flush();
2647 ASSERT_OK(s);
2648
2649 listener->OverrideBGError(Status(error_msg, Status::Severity::kHardError));
2650 listener->EnableAutoRecovery(false);
2651 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency(
2652 {{"DBImpl::FlushMemTable:FlushMemTableFinished",
2653 "BackgroundCallCompaction:0"},
2654 {"CompactionJob::FinishCompactionOutputFile1",
2655 "CompactionWriteRetryableErrorAutoRecover0"}});
2656 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
2657 "DBImpl::BackgroundCompaction:Start",
2658 [&](void*) { fault_fs_->SetFilesystemActive(true); });
2659 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
2660 "BackgroundCallCompaction:0", [&](void*) { fail_first.store(true); });
2661 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
2662 "CompactionJob::Run():PausingManualCompaction:2", [&](void*) {
2663 if (fail_first.load() && fail_second.load()) {
2664 fault_fs_->SetFilesystemActive(false, error_msg);
2665 fail_second.store(false);
2666 }
2667 });
2668 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
2669
2670 ASSERT_OK(Put(Key(1), "val"));
2671 s = Flush();
2672 ASSERT_OK(s);
2673
2674 s = dbfull()->TEST_WaitForCompact();
2675 ASSERT_OK(s);
2676 TEST_SYNC_POINT("CompactionWriteRetryableErrorAutoRecover0");
2677 SyncPoint::GetInstance()->ClearAllCallBacks();
2678 SyncPoint::GetInstance()->DisableProcessing();
2679
2680 Reopen(GetDefaultOptions());
2681}
2682
20effc67
TL		 2683class DBErrorHandlingFencingTest : public DBErrorHandlingFSTest,
2684 public testing::WithParamInterface<bool> {};
2685
2686TEST_P(DBErrorHandlingFencingTest, FLushWriteFenced) {
2687 std::shared_ptr<ErrorHandlerFSListener> listener(
2688 new ErrorHandlerFSListener());
2689 Options options = GetDefaultOptions();
2690 options.env = fault_env_.get();
2691 options.create_if_missing = true;
2692 options.listeners.emplace_back(listener);
2693 options.paranoid_checks = GetParam();
2694 Status s;
2695
2696 listener->EnableAutoRecovery(true);
2697 DestroyAndReopen(options);
2698
2699 ASSERT_OK(Put(Key(0), "val"));
2700 SyncPoint::GetInstance()->SetCallBack("FlushJob::Start", [&](void*) {
2701 fault_fs_->SetFilesystemActive(false, IOStatus::IOFenced("IO fenced"));
2702 });
2703 SyncPoint::GetInstance()->EnableProcessing();
2704 s = Flush();
2705 ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kFatalError);
2706 ASSERT_TRUE(s.IsIOFenced());
2707 SyncPoint::GetInstance()->DisableProcessing();
2708 fault_fs_->SetFilesystemActive(true);
2709 s = dbfull()->Resume();
2710 ASSERT_TRUE(s.IsIOFenced());
2711 Destroy(options);
2712}
2713
2714TEST_P(DBErrorHandlingFencingTest, ManifestWriteFenced) {
2715 std::shared_ptr<ErrorHandlerFSListener> listener(
2716 new ErrorHandlerFSListener());
2717 Options options = GetDefaultOptions();
2718 options.env = fault_env_.get();
2719 options.create_if_missing = true;
2720 options.listeners.emplace_back(listener);
2721 options.paranoid_checks = GetParam();
2722 Status s;
2723 std::string old_manifest;
2724 std::string new_manifest;
2725
2726 listener->EnableAutoRecovery(true);
2727 DestroyAndReopen(options);
2728 old_manifest = GetManifestNameFromLiveFiles();
2729
2730 ASSERT_OK(Put(Key(0), "val"));
1e59de90 2731 ASSERT_OK(Flush());
20effc67
TL		 2732 ASSERT_OK(Put(Key(1), "val"));
2733 SyncPoint::GetInstance()->SetCallBack(
2734 "VersionSet::LogAndApply:WriteManifest", [&](void*) {
2735 fault_fs_->SetFilesystemActive(false, IOStatus::IOFenced("IO fenced"));
2736 });
2737 SyncPoint::GetInstance()->EnableProcessing();
2738 s = Flush();
2739 ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kFatalError);
2740 ASSERT_TRUE(s.IsIOFenced());
2741 SyncPoint::GetInstance()->ClearAllCallBacks();
2742 SyncPoint::GetInstance()->DisableProcessing();
2743 fault_fs_->SetFilesystemActive(true);
2744 s = dbfull()->Resume();
2745 ASSERT_TRUE(s.IsIOFenced());
2746 Close();
2747}
2748
2749TEST_P(DBErrorHandlingFencingTest, CompactionWriteFenced) {
2750 std::shared_ptr<ErrorHandlerFSListener> listener(
2751 new ErrorHandlerFSListener());
2752 Options options = GetDefaultOptions();
2753 options.env = fault_env_.get();
2754 options.create_if_missing = true;
2755 options.level0_file_num_compaction_trigger = 2;
2756 options.listeners.emplace_back(listener);
2757 options.paranoid_checks = GetParam();
2758 Status s;
2759 DestroyAndReopen(options);
2760
2761 ASSERT_OK(Put(Key(0), "va;"));
2762 ASSERT_OK(Put(Key(2), "va;"));
2763 s = Flush();
1e59de90 2764 ASSERT_OK(s);
20effc67
TL		 2765
2766 listener->EnableAutoRecovery(true);
2767 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency(
2768 {{"DBImpl::FlushMemTable:FlushMemTableFinished",
2769 "BackgroundCallCompaction:0"}});
2770 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
2771 "BackgroundCallCompaction:0", [&](void*) {
2772 fault_fs_->SetFilesystemActive(false, IOStatus::IOFenced("IO fenced"));
2773 });
2774 ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
2775
2776 ASSERT_OK(Put(Key(1), "val"));
2777 s = Flush();
1e59de90 2778 ASSERT_OK(s);
20effc67
TL		 2779
2780 s = dbfull()->TEST_WaitForCompact();
2781 ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kFatalError);
2782 ASSERT_TRUE(s.IsIOFenced());
2783
2784 fault_fs_->SetFilesystemActive(true);
2785 s = dbfull()->Resume();
2786 ASSERT_TRUE(s.IsIOFenced());
2787 Destroy(options);
2788}
2789
2790TEST_P(DBErrorHandlingFencingTest, WALWriteFenced) {
2791 std::shared_ptr<ErrorHandlerFSListener> listener(
2792 new ErrorHandlerFSListener());
2793 Options options = GetDefaultOptions();
2794 options.env = fault_env_.get();
2795 options.create_if_missing = true;
2796 options.writable_file_max_buffer_size = 32768;
2797 options.listeners.emplace_back(listener);
2798 options.paranoid_checks = GetParam();
2799 Status s;
2800 Random rnd(301);
2801
2802 listener->EnableAutoRecovery(true);
2803 DestroyAndReopen(options);
2804
2805 {
2806 WriteBatch batch;
2807
2808 for (auto i = 0; i < 100; ++i) {
2809 ASSERT_OK(batch.Put(Key(i), rnd.RandomString(1024)));
2810 }
2811
2812 WriteOptions wopts;
2813 wopts.sync = true;
1e59de90 2814 ASSERT_OK(dbfull()->Write(wopts, &batch));
20effc67
TL		 2815 };
2816
2817 {
2818 WriteBatch batch;
2819 int write_error = 0;
2820
2821 for (auto i = 100; i < 199; ++i) {
2822 ASSERT_OK(batch.Put(Key(i), rnd.RandomString(1024)));
2823 }
2824
2825 SyncPoint::GetInstance()->SetCallBack(
2826 "WritableFileWriter::Append:BeforePrepareWrite", [&](void*) {
2827 write_error++;
2828 if (write_error > 2) {
2829 fault_fs_->SetFilesystemActive(false,
2830 IOStatus::IOFenced("IO fenced"));
2831 }
2832 });
2833 SyncPoint::GetInstance()->EnableProcessing();
2834 WriteOptions wopts;
2835 wopts.sync = true;
2836 s = dbfull()->Write(wopts, &batch);
2837 ASSERT_TRUE(s.IsIOFenced());
2838 }
2839 SyncPoint::GetInstance()->DisableProcessing();
2840 fault_fs_->SetFilesystemActive(true);
2841 {
2842 WriteBatch batch;
2843
2844 for (auto i = 0; i < 100; ++i) {
2845 ASSERT_OK(batch.Put(Key(i), rnd.RandomString(1024)));
2846 }
2847
2848 WriteOptions wopts;
2849 wopts.sync = true;
2850 s = dbfull()->Write(wopts, &batch);
2851 ASSERT_TRUE(s.IsIOFenced());
2852 }
2853 Close();
2854}
2855
2856INSTANTIATE_TEST_CASE_P(DBErrorHandlingFSTest, DBErrorHandlingFencingTest,
2857 ::testing::Bool());
2858
2859} // namespace ROCKSDB_NAMESPACE
2860
2861int main(int argc, char** argv) {
2862 ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
2863 ::testing::InitGoogleTest(&argc, argv);
2864 return RUN_ALL_TESTS();
2865}
2866
2867#else
2868#include <stdio.h>
2869
2870int main(int /*argc*/, char** /*argv*/) {
2871 fprintf(stderr, "SKIPPED as Cuckoo table is not supported in ROCKSDB_LITE\n");
2872 return 0;
2873}
2874
2875#endif // ROCKSDB_LITE
Ceph