ceph/src/rocksdb/util/delete_scheduler_test.cc

   1 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
   2 //  This source code is licensed under the BSD-style license found in the
   3 //  LICENSE file in the root directory of this source tree. An additional grant
   4 //  of patent rights can be found in the PATENTS file in the same directory.
   5
   6 #ifndef __STDC_FORMAT_MACROS
   7 #define __STDC_FORMAT_MACROS
   8 #endif
   9
  10 #include <inttypes.h>
  11 #include <atomic>
  12 #include <thread>
  13 #include <vector>
  14
  15 #include "rocksdb/env.h"
  16 #include "rocksdb/options.h"
  17 #include "util/delete_scheduler.h"
  18 #include "util/string_util.h"
  19 #include "util/sync_point.h"
  20 #include "util/testharness.h"
  21 #include "util/testutil.h"
  22
  23 #ifndef ROCKSDB_LITE
  24
  25 namespace rocksdb {
  26
  27 class DeleteSchedulerTest : public testing::Test {
  28  public:
  29   DeleteSchedulerTest() : env_(Env::Default()) {
  30     dummy_files_dir_ = test::TmpDir(env_) + "/delete_scheduler_dummy_data_dir";
  31     DestroyAndCreateDir(dummy_files_dir_);
  32     trash_dir_ = test::TmpDir(env_) + "/delete_scheduler_trash";
  33     DestroyAndCreateDir(trash_dir_);
  34   }
  35
  36   ~DeleteSchedulerTest() {
  37     rocksdb::SyncPoint::GetInstance()->DisableProcessing();
  38     rocksdb::SyncPoint::GetInstance()->LoadDependency({});
  39     rocksdb::SyncPoint::GetInstance()->ClearAllCallBacks();
  40     test::DestroyDir(env_, dummy_files_dir_);
  41   }
  42
  43   void DestroyAndCreateDir(const std::string& dir) {
  44     ASSERT_OK(test::DestroyDir(env_, dir));
  45     EXPECT_OK(env_->CreateDir(dir));
  46   }
  47
  48   int CountFilesInDir(const std::string& dir) {
  49     std::vector<std::string> files_in_dir;
  50     EXPECT_OK(env_->GetChildren(dir, &files_in_dir));
  51     // Ignore "." and ".."
  52     return static_cast<int>(files_in_dir.size()) - 2;
  53   }
  54
  55   std::string NewDummyFile(const std::string& file_name, uint64_t size = 1024) {
  56     std::string file_path = dummy_files_dir_ + "/" + file_name;
  57     std::unique_ptr<WritableFile> f;
  58     env_->NewWritableFile(file_path, &f, EnvOptions());
  59     std::string data(size, 'A');
  60     EXPECT_OK(f->Append(data));
  61     EXPECT_OK(f->Close());
  62     return file_path;
  63   }
  64
  65   void NewDeleteScheduler() {
  66     ASSERT_OK(env_->CreateDirIfMissing(trash_dir_));
  67     delete_scheduler_.reset(new DeleteScheduler(
  68         env_, trash_dir_, rate_bytes_per_sec_, nullptr, nullptr));
  69   }
  70
  71   Env* env_;
  72   std::string dummy_files_dir_;
  73   std::string trash_dir_;
  74   int64_t rate_bytes_per_sec_;
  75   std::shared_ptr<DeleteScheduler> delete_scheduler_;
  76 };
  77
  78 // Test the basic functionality of DeleteScheduler (Rate Limiting).
  79 // 1- Create 100 dummy files
  80 // 2- Delete the 100 dummy files using DeleteScheduler
  81 // --- Hold DeleteScheduler::BackgroundEmptyTrash ---
  82 // 3- Wait for DeleteScheduler to delete all files in trash
  83 // 4- Verify that BackgroundEmptyTrash used to correct penlties for the files
  84 // 5- Make sure that all created files were completely deleted
  85 TEST_F(DeleteSchedulerTest, BasicRateLimiting) {
  86   rocksdb::SyncPoint::GetInstance()->LoadDependency({
  87       {"DeleteSchedulerTest::BasicRateLimiting:1",
  88        "DeleteScheduler::BackgroundEmptyTrash"},
  89   });
  90
  91   std::vector<uint64_t> penalties;
  92   rocksdb::SyncPoint::GetInstance()->SetCallBack(
  93       "DeleteScheduler::BackgroundEmptyTrash:Wait",
  94       [&](void* arg) { penalties.push_back(*(static_cast<uint64_t*>(arg))); });
  95
  96   int num_files = 100;  // 100 files
  97   uint64_t file_size = 1024;  // every file is 1 kb
  98   std::vector<uint64_t> delete_kbs_per_sec = {512, 200, 100, 50, 25};
  99
 100   for (size_t t = 0; t < delete_kbs_per_sec.size(); t++) {
 101     penalties.clear();
 102     rocksdb::SyncPoint::GetInstance()->ClearTrace();
 103     rocksdb::SyncPoint::GetInstance()->EnableProcessing();
 104
 105     DestroyAndCreateDir(dummy_files_dir_);
 106     rate_bytes_per_sec_ = delete_kbs_per_sec[t] * 1024;
 107     NewDeleteScheduler();
 108
 109     // Create 100 dummy files, every file is 1 Kb
 110     std::vector<std::string> generated_files;
 111     for (int i = 0; i < num_files; i++) {
 112       std::string file_name = "file" + ToString(i) + ".data";
 113       generated_files.push_back(NewDummyFile(file_name, file_size));
 114     }
 115
 116     // Delete dummy files and measure time spent to empty trash
 117     for (int i = 0; i < num_files; i++) {
 118       ASSERT_OK(delete_scheduler_->DeleteFile(generated_files[i]));
 119     }
 120     ASSERT_EQ(CountFilesInDir(dummy_files_dir_), 0);
 121
 122     uint64_t delete_start_time = env_->NowMicros();
 123     TEST_SYNC_POINT("DeleteSchedulerTest::BasicRateLimiting:1");
 124     delete_scheduler_->WaitForEmptyTrash();
 125     uint64_t time_spent_deleting = env_->NowMicros() - delete_start_time;
 126
 127     auto bg_errors = delete_scheduler_->GetBackgroundErrors();
 128     ASSERT_EQ(bg_errors.size(), 0);
 129
 130     uint64_t total_files_size = 0;
 131     uint64_t expected_penlty = 0;
 132     ASSERT_EQ(penalties.size(), num_files);
 133     for (int i = 0; i < num_files; i++) {
 134       total_files_size += file_size;
 135       expected_penlty = ((total_files_size * 1000000) / rate_bytes_per_sec_);
 136       ASSERT_EQ(expected_penlty, penalties[i]);
 137     }
 138     ASSERT_GT(time_spent_deleting, expected_penlty * 0.9);
 139
 140     ASSERT_EQ(CountFilesInDir(trash_dir_), 0);
 141     rocksdb::SyncPoint::GetInstance()->DisableProcessing();
 142   }
 143 }
 144
 145 // Same as the BasicRateLimiting test but delete files in multiple threads.
 146 // 1- Create 100 dummy files
 147 // 2- Delete the 100 dummy files using DeleteScheduler using 10 threads
 148 // --- Hold DeleteScheduler::BackgroundEmptyTrash ---
 149 // 3- Wait for DeleteScheduler to delete all files in queue
 150 // 4- Verify that BackgroundEmptyTrash used to correct penlties for the files
 151 // 5- Make sure that all created files were completely deleted
 152 TEST_F(DeleteSchedulerTest, RateLimitingMultiThreaded) {
 153   rocksdb::SyncPoint::GetInstance()->LoadDependency({
 154       {"DeleteSchedulerTest::RateLimitingMultiThreaded:1",
 155        "DeleteScheduler::BackgroundEmptyTrash"},
 156   });
 157
 158   std::vector<uint64_t> penalties;
 159   rocksdb::SyncPoint::GetInstance()->SetCallBack(
 160       "DeleteScheduler::BackgroundEmptyTrash:Wait",
 161       [&](void* arg) { penalties.push_back(*(static_cast<uint64_t*>(arg))); });
 162
 163   int thread_cnt = 10;
 164   int num_files = 10;  // 10 files per thread
 165   uint64_t file_size = 1024;  // every file is 1 kb
 166
 167   std::vector<uint64_t> delete_kbs_per_sec = {512, 200, 100, 50, 25};
 168   for (size_t t = 0; t < delete_kbs_per_sec.size(); t++) {
 169     penalties.clear();
 170     rocksdb::SyncPoint::GetInstance()->ClearTrace();
 171     rocksdb::SyncPoint::GetInstance()->EnableProcessing();
 172
 173     DestroyAndCreateDir(dummy_files_dir_);
 174     rate_bytes_per_sec_ = delete_kbs_per_sec[t] * 1024;
 175     NewDeleteScheduler();
 176
 177     // Create 100 dummy files, every file is 1 Kb
 178     std::vector<std::string> generated_files;
 179     for (int i = 0; i < num_files * thread_cnt; i++) {
 180       std::string file_name = "file" + ToString(i) + ".data";
 181       generated_files.push_back(NewDummyFile(file_name, file_size));
 182     }
 183
 184     // Delete dummy files using 10 threads and measure time spent to empty trash
 185     std::atomic<int> thread_num(0);
 186     std::vector<port::Thread> threads;
 187     std::function<void()> delete_thread = [&]() {
 188       int idx = thread_num.fetch_add(1);
 189       int range_start = idx * num_files;
 190       int range_end = range_start + num_files;
 191       for (int j = range_start; j < range_end; j++) {
 192         ASSERT_OK(delete_scheduler_->DeleteFile(generated_files[j]));
 193       }
 194     };
 195
 196     for (int i = 0; i < thread_cnt; i++) {
 197       threads.emplace_back(delete_thread);
 198     }
 199
 200     for (size_t i = 0; i < threads.size(); i++) {
 201       threads[i].join();
 202     }
 203
 204     uint64_t delete_start_time = env_->NowMicros();
 205     TEST_SYNC_POINT("DeleteSchedulerTest::RateLimitingMultiThreaded:1");
 206     delete_scheduler_->WaitForEmptyTrash();
 207     uint64_t time_spent_deleting = env_->NowMicros() - delete_start_time;
 208
 209     auto bg_errors = delete_scheduler_->GetBackgroundErrors();
 210     ASSERT_EQ(bg_errors.size(), 0);
 211
 212     uint64_t total_files_size = 0;
 213     uint64_t expected_penlty = 0;
 214     ASSERT_EQ(penalties.size(), num_files * thread_cnt);
 215     for (int i = 0; i < num_files * thread_cnt; i++) {
 216       total_files_size += file_size;
 217       expected_penlty = ((total_files_size * 1000000) / rate_bytes_per_sec_);
 218       ASSERT_EQ(expected_penlty, penalties[i]);
 219     }
 220     ASSERT_GT(time_spent_deleting, expected_penlty * 0.9);
 221
 222     ASSERT_EQ(CountFilesInDir(dummy_files_dir_), 0);
 223     ASSERT_EQ(CountFilesInDir(trash_dir_), 0);
 224     rocksdb::SyncPoint::GetInstance()->DisableProcessing();
 225   }
 226 }
 227
 228 // Disable rate limiting by setting rate_bytes_per_sec_ to 0 and make sure
 229 // that when DeleteScheduler delete a file it delete it immediately and dont
 230 // move it to trash
 231 TEST_F(DeleteSchedulerTest, DisableRateLimiting) {
 232   int bg_delete_file = 0;
 233   rocksdb::SyncPoint::GetInstance()->SetCallBack(
 234       "DeleteScheduler::DeleteTrashFile:DeleteFile",
 235       [&](void* arg) { bg_delete_file++; });
 236
 237   rocksdb::SyncPoint::GetInstance()->EnableProcessing();
 238
 239   rate_bytes_per_sec_ = 0;
 240   NewDeleteScheduler();
 241
 242   for (int i = 0; i < 10; i++) {
 243     // Every file we delete will be deleted immediately
 244     std::string dummy_file = NewDummyFile("dummy.data");
 245     ASSERT_OK(delete_scheduler_->DeleteFile(dummy_file));
 246     ASSERT_TRUE(env_->FileExists(dummy_file).IsNotFound());
 247     ASSERT_EQ(CountFilesInDir(dummy_files_dir_), 0);
 248     ASSERT_EQ(CountFilesInDir(trash_dir_), 0);
 249   }
 250
 251   ASSERT_EQ(bg_delete_file, 0);
 252
 253   rocksdb::SyncPoint::GetInstance()->DisableProcessing();
 254 }
 255
 256 // Testing that moving files to trash with the same name is not a problem
 257 // 1- Create 10 files with the same name "conflict.data"
 258 // 2- Delete the 10 files using DeleteScheduler
 259 // 3- Make sure that trash directory contain 10 files ("conflict.data" x 10)
 260 // --- Hold DeleteScheduler::BackgroundEmptyTrash ---
 261 // 4- Make sure that files are deleted from trash
 262 TEST_F(DeleteSchedulerTest, ConflictNames) {
 263   rocksdb::SyncPoint::GetInstance()->LoadDependency({
 264       {"DeleteSchedulerTest::ConflictNames:1",
 265        "DeleteScheduler::BackgroundEmptyTrash"},
 266   });
 267   rocksdb::SyncPoint::GetInstance()->EnableProcessing();
 268
 269   rate_bytes_per_sec_ = 1024 * 1024;  // 1 Mb/sec
 270   NewDeleteScheduler();
 271
 272   // Create "conflict.data" and move it to trash 10 times
 273   for (int i = 0; i < 10; i++) {
 274     std::string dummy_file = NewDummyFile("conflict.data");
 275     ASSERT_OK(delete_scheduler_->DeleteFile(dummy_file));
 276   }
 277   ASSERT_EQ(CountFilesInDir(dummy_files_dir_), 0);
 278   // 10 files ("conflict.data" x 10) in trash
 279   ASSERT_EQ(CountFilesInDir(trash_dir_), 10);
 280
 281   // Hold BackgroundEmptyTrash
 282   TEST_SYNC_POINT("DeleteSchedulerTest::ConflictNames:1");
 283   delete_scheduler_->WaitForEmptyTrash();
 284   ASSERT_EQ(CountFilesInDir(trash_dir_), 0);
 285
 286   auto bg_errors = delete_scheduler_->GetBackgroundErrors();
 287   ASSERT_EQ(bg_errors.size(), 0);
 288
 289   rocksdb::SyncPoint::GetInstance()->DisableProcessing();
 290 }
 291
 292 // 1- Create 10 dummy files
 293 // 2- Delete the 10 files using DeleteScheduler (move them to trsah)
 294 // 3- Delete the 10 files directly (using env_->DeleteFile)
 295 // --- Hold DeleteScheduler::BackgroundEmptyTrash ---
 296 // 4- Make sure that DeleteScheduler failed to delete the 10 files and
 297 //    reported 10 background errors
 298 TEST_F(DeleteSchedulerTest, BackgroundError) {
 299   rocksdb::SyncPoint::GetInstance()->LoadDependency({
 300       {"DeleteSchedulerTest::BackgroundError:1",
 301        "DeleteScheduler::BackgroundEmptyTrash"},
 302   });
 303   rocksdb::SyncPoint::GetInstance()->EnableProcessing();
 304
 305   rate_bytes_per_sec_ = 1024 * 1024;  // 1 Mb/sec
 306   NewDeleteScheduler();
 307
 308   // Generate 10 dummy files and move them to trash
 309   for (int i = 0; i < 10; i++) {
 310     std::string file_name = "data_" + ToString(i) + ".data";
 311     ASSERT_OK(delete_scheduler_->DeleteFile(NewDummyFile(file_name)));
 312   }
 313   ASSERT_EQ(CountFilesInDir(dummy_files_dir_), 0);
 314   ASSERT_EQ(CountFilesInDir(trash_dir_), 10);
 315
 316   // Delete 10 files from trash, this will cause background errors in
 317   // BackgroundEmptyTrash since we already deleted the files it was
 318   // goind to delete
 319   for (int i = 0; i < 10; i++) {
 320     std::string file_name = "data_" + ToString(i) + ".data";
 321     ASSERT_OK(env_->DeleteFile(trash_dir_ + "/" + file_name));
 322   }
 323
 324   // Hold BackgroundEmptyTrash
 325   TEST_SYNC_POINT("DeleteSchedulerTest::BackgroundError:1");
 326   delete_scheduler_->WaitForEmptyTrash();
 327   auto bg_errors = delete_scheduler_->GetBackgroundErrors();
 328   ASSERT_EQ(bg_errors.size(), 10);
 329
 330   rocksdb::SyncPoint::GetInstance()->DisableProcessing();
 331 }
 332
 333 // 1- Create 10 dummy files
 334 // 2- Delete 10 dummy files using DeleteScheduler
 335 // 3- Wait for DeleteScheduler to delete all files in queue
 336 // 4- Make sure all files in trash directory were deleted
 337 // 5- Repeat previous steps 5 times
 338 TEST_F(DeleteSchedulerTest, StartBGEmptyTrashMultipleTimes) {
 339   int bg_delete_file = 0;
 340   rocksdb::SyncPoint::GetInstance()->SetCallBack(
 341       "DeleteScheduler::DeleteTrashFile:DeleteFile",
 342       [&](void* arg) { bg_delete_file++; });
 343   rocksdb::SyncPoint::GetInstance()->EnableProcessing();
 344
 345   rate_bytes_per_sec_ = 1024 * 1024;  // 1 MB / sec
 346   NewDeleteScheduler();
 347
 348   // Move files to trash, wait for empty trash, start again
 349   for (int run = 1; run <= 5; run++) {
 350     // Generate 10 dummy files and move them to trash
 351     for (int i = 0; i < 10; i++) {
 352       std::string file_name = "data_" + ToString(i) + ".data";
 353       ASSERT_OK(delete_scheduler_->DeleteFile(NewDummyFile(file_name)));
 354     }
 355     ASSERT_EQ(CountFilesInDir(dummy_files_dir_), 0);
 356     delete_scheduler_->WaitForEmptyTrash();
 357     ASSERT_EQ(bg_delete_file, 10 * run);
 358     ASSERT_EQ(CountFilesInDir(trash_dir_), 0);
 359
 360     auto bg_errors = delete_scheduler_->GetBackgroundErrors();
 361     ASSERT_EQ(bg_errors.size(), 0);
 362   }
 363
 364   ASSERT_EQ(bg_delete_file, 50);
 365   rocksdb::SyncPoint::GetInstance()->EnableProcessing();
 366 }
 367
 368 // 1- Create a DeleteScheduler with very slow rate limit (1 Byte / sec)
 369 // 2- Delete 100 files using DeleteScheduler
 370 // 3- Delete the DeleteScheduler (call the destructor while queue is not empty)
 371 // 4- Make sure that not all files were deleted from trash and that
 372 //    DeleteScheduler background thread did not delete all files
 373 TEST_F(DeleteSchedulerTest, DestructorWithNonEmptyQueue) {
 374   int bg_delete_file = 0;
 375   rocksdb::SyncPoint::GetInstance()->SetCallBack(
 376       "DeleteScheduler::DeleteTrashFile:DeleteFile",
 377       [&](void* arg) { bg_delete_file++; });
 378   rocksdb::SyncPoint::GetInstance()->EnableProcessing();
 379
 380   rate_bytes_per_sec_ = 1;  // 1 Byte / sec
 381   NewDeleteScheduler();
 382
 383   for (int i = 0; i < 100; i++) {
 384     std::string file_name = "data_" + ToString(i) + ".data";
 385     ASSERT_OK(delete_scheduler_->DeleteFile(NewDummyFile(file_name)));
 386   }
 387
 388   // Deleting 100 files will need >28 hours to delete
 389   // we will delete the DeleteScheduler while delete queue is not empty
 390   delete_scheduler_.reset();
 391
 392   ASSERT_LT(bg_delete_file, 100);
 393   ASSERT_GT(CountFilesInDir(trash_dir_), 0);
 394
 395   rocksdb::SyncPoint::GetInstance()->DisableProcessing();
 396 }
 397
 398 // 1- Delete the trash directory
 399 // 2- Delete 10 files using DeleteScheduler
 400 // 3- Make sure that the 10 files were deleted immediately since DeleteScheduler
 401 //    failed to move them to trash directory
 402 TEST_F(DeleteSchedulerTest, MoveToTrashError) {
 403   int bg_delete_file = 0;
 404   rocksdb::SyncPoint::GetInstance()->SetCallBack(
 405       "DeleteScheduler::DeleteTrashFile:DeleteFile",
 406       [&](void* arg) { bg_delete_file++; });
 407   rocksdb::SyncPoint::GetInstance()->EnableProcessing();
 408
 409   rate_bytes_per_sec_ = 1024;  // 1 Kb / sec
 410   NewDeleteScheduler();
 411
 412   // We will delete the trash directory, that mean that DeleteScheduler wont
 413   // be able to move files to trash and will delete files them immediately.
 414   ASSERT_OK(test::DestroyDir(env_, trash_dir_));
 415   for (int i = 0; i < 10; i++) {
 416     std::string file_name = "data_" + ToString(i) + ".data";
 417     ASSERT_OK(delete_scheduler_->DeleteFile(NewDummyFile(file_name)));
 418   }
 419
 420   ASSERT_EQ(CountFilesInDir(dummy_files_dir_), 0);
 421   ASSERT_EQ(bg_delete_file, 0);
 422
 423   rocksdb::SyncPoint::GetInstance()->DisableProcessing();
 424 }
 425
 426 TEST_F(DeleteSchedulerTest, DISABLED_DynamicRateLimiting1) {
 427   std::vector<uint64_t> penalties;
 428   int bg_delete_file = 0;
 429   int fg_delete_file = 0;
 430   rocksdb::SyncPoint::GetInstance()->SetCallBack(
 431       "DeleteScheduler::DeleteTrashFile:DeleteFile",
 432       [&](void* arg) { bg_delete_file++; });
 433   rocksdb::SyncPoint::GetInstance()->SetCallBack(
 434       "DeleteScheduler::DeleteFile",
 435       [&](void* arg) { fg_delete_file++; });
 436   rocksdb::SyncPoint::GetInstance()->SetCallBack(
 437       "DeleteScheduler::BackgroundEmptyTrash:Wait",
 438       [&](void* arg) { penalties.push_back(*(static_cast<int*>(arg))); });
 439
 440   rocksdb::SyncPoint::GetInstance()->LoadDependency({
 441       {"DeleteSchedulerTest::DynamicRateLimiting1:1",
 442        "DeleteScheduler::BackgroundEmptyTrash"},
 443   });
 444   rocksdb::SyncPoint::GetInstance()->EnableProcessing();
 445
 446   rate_bytes_per_sec_ = 0;  // Disable rate limiting initially
 447   NewDeleteScheduler();
 448
 449
 450   int num_files = 10;  // 10 files
 451   uint64_t file_size = 1024;  // every file is 1 kb
 452
 453   std::vector<int64_t> delete_kbs_per_sec = {512, 200, 0, 100, 50, -2, 25};
 454   for (size_t t = 0; t < delete_kbs_per_sec.size(); t++) {
 455     penalties.clear();
 456     bg_delete_file = 0;
 457     fg_delete_file = 0;
 458     rocksdb::SyncPoint::GetInstance()->ClearTrace();
 459     rocksdb::SyncPoint::GetInstance()->EnableProcessing();
 460
 461     DestroyAndCreateDir(dummy_files_dir_);
 462     rate_bytes_per_sec_ = delete_kbs_per_sec[t] * 1024;
 463     delete_scheduler_->SetRateBytesPerSecond(rate_bytes_per_sec_);
 464
 465     // Create 100 dummy files, every file is 1 Kb
 466     std::vector<std::string> generated_files;
 467     for (int i = 0; i < num_files; i++) {
 468       std::string file_name = "file" + ToString(i) + ".data";
 469       generated_files.push_back(NewDummyFile(file_name, file_size));
 470     }
 471
 472     // Delete dummy files and measure time spent to empty trash
 473     for (int i = 0; i < num_files; i++) {
 474       ASSERT_OK(delete_scheduler_->DeleteFile(generated_files[i]));
 475     }
 476     ASSERT_EQ(CountFilesInDir(dummy_files_dir_), 0);
 477
 478     if (rate_bytes_per_sec_ > 0) {
 479       uint64_t delete_start_time = env_->NowMicros();
 480       TEST_SYNC_POINT("DeleteSchedulerTest::DynamicRateLimiting1:1");
 481       delete_scheduler_->WaitForEmptyTrash();
 482       uint64_t time_spent_deleting = env_->NowMicros() - delete_start_time;
 483
 484       auto bg_errors = delete_scheduler_->GetBackgroundErrors();
 485       ASSERT_EQ(bg_errors.size(), 0);
 486
 487       uint64_t total_files_size = 0;
 488       uint64_t expected_penlty = 0;
 489       ASSERT_EQ(penalties.size(), num_files);
 490       for (int i = 0; i < num_files; i++) {
 491         total_files_size += file_size;
 492         expected_penlty = ((total_files_size * 1000000) / rate_bytes_per_sec_);
 493         ASSERT_EQ(expected_penlty, penalties[i]);
 494       }
 495       ASSERT_GT(time_spent_deleting, expected_penlty * 0.9);
 496       ASSERT_EQ(bg_delete_file, num_files);
 497       ASSERT_EQ(fg_delete_file, 0);
 498     } else {
 499       ASSERT_EQ(penalties.size(), 0);
 500       ASSERT_EQ(bg_delete_file, 0);
 501       ASSERT_EQ(fg_delete_file, num_files);
 502     }
 503
 504     ASSERT_EQ(CountFilesInDir(trash_dir_), 0);
 505     rocksdb::SyncPoint::GetInstance()->DisableProcessing();
 506   }
 507 }
 508
 509 }  // namespace rocksdb
 510
 511 int main(int argc, char** argv) {
 512   ::testing::InitGoogleTest(&argc, argv);
 513   return RUN_ALL_TESTS();
 514 }
 515
 516 #else
 517 int main(int argc, char** argv) {
 518   printf("DeleteScheduler is not supported in ROCKSDB_LITE\n");
 519   return 0;
 520 }
 521 #endif  // ROCKSDB_LITE