]>
Commit | Line | Data |
---|---|---|
7c673cae | 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. |
11fdf7f2 TL |
2 | // This source code is licensed under both the GPLv2 (found in the |
3 | // COPYING file in the root directory) and Apache 2.0 License | |
4 | // (found in the LICENSE.Apache file in the root directory). | |
7c673cae FG |
5 | |
6 | #ifndef __STDC_FORMAT_MACROS | |
7 | #define __STDC_FORMAT_MACROS | |
8 | #endif | |
9 | ||
10 | #include <inttypes.h> | |
11 | #include <atomic> | |
12 | #include <thread> | |
13 | #include <vector> | |
14 | ||
15 | #include "rocksdb/env.h" | |
16 | #include "rocksdb/options.h" | |
17 | #include "util/delete_scheduler.h" | |
11fdf7f2 | 18 | #include "util/sst_file_manager_impl.h" |
7c673cae FG |
19 | #include "util/string_util.h" |
20 | #include "util/sync_point.h" | |
21 | #include "util/testharness.h" | |
22 | #include "util/testutil.h" | |
23 | ||
24 | #ifndef ROCKSDB_LITE | |
25 | ||
26 | namespace rocksdb { | |
27 | ||
28 | class DeleteSchedulerTest : public testing::Test { | |
29 | public: | |
30 | DeleteSchedulerTest() : env_(Env::Default()) { | |
11fdf7f2 TL |
31 | const int kNumDataDirs = 3; |
32 | dummy_files_dirs_.reserve(kNumDataDirs); | |
33 | for (size_t i = 0; i < kNumDataDirs; ++i) { | |
34 | dummy_files_dirs_.emplace_back( | |
35 | test::PerThreadDBPath(env_, "delete_scheduler_dummy_data_dir") + | |
36 | ToString(i)); | |
37 | DestroyAndCreateDir(dummy_files_dirs_.back()); | |
38 | } | |
7c673cae FG |
39 | } |
40 | ||
494da23a | 41 | ~DeleteSchedulerTest() override { |
7c673cae FG |
42 | rocksdb::SyncPoint::GetInstance()->DisableProcessing(); |
43 | rocksdb::SyncPoint::GetInstance()->LoadDependency({}); | |
44 | rocksdb::SyncPoint::GetInstance()->ClearAllCallBacks(); | |
11fdf7f2 TL |
45 | for (const auto& dummy_files_dir : dummy_files_dirs_) { |
46 | test::DestroyDir(env_, dummy_files_dir); | |
47 | } | |
7c673cae FG |
48 | } |
49 | ||
50 | void DestroyAndCreateDir(const std::string& dir) { | |
51 | ASSERT_OK(test::DestroyDir(env_, dir)); | |
52 | EXPECT_OK(env_->CreateDir(dir)); | |
53 | } | |
54 | ||
11fdf7f2 TL |
55 | int CountNormalFiles(size_t dummy_files_dirs_idx = 0) { |
56 | std::vector<std::string> files_in_dir; | |
57 | EXPECT_OK(env_->GetChildren(dummy_files_dirs_[dummy_files_dirs_idx], | |
58 | &files_in_dir)); | |
59 | ||
60 | int normal_cnt = 0; | |
61 | for (auto& f : files_in_dir) { | |
62 | if (!DeleteScheduler::IsTrashFile(f) && f != "." && f != "..") { | |
63 | normal_cnt++; | |
64 | } | |
65 | } | |
66 | return normal_cnt; | |
67 | } | |
68 | ||
69 | int CountTrashFiles(size_t dummy_files_dirs_idx = 0) { | |
7c673cae | 70 | std::vector<std::string> files_in_dir; |
11fdf7f2 TL |
71 | EXPECT_OK(env_->GetChildren(dummy_files_dirs_[dummy_files_dirs_idx], |
72 | &files_in_dir)); | |
73 | ||
74 | int trash_cnt = 0; | |
75 | for (auto& f : files_in_dir) { | |
76 | if (DeleteScheduler::IsTrashFile(f)) { | |
77 | trash_cnt++; | |
78 | } | |
79 | } | |
80 | return trash_cnt; | |
7c673cae FG |
81 | } |
82 | ||
11fdf7f2 TL |
83 | std::string NewDummyFile(const std::string& file_name, uint64_t size = 1024, |
84 | size_t dummy_files_dirs_idx = 0) { | |
85 | std::string file_path = | |
86 | dummy_files_dirs_[dummy_files_dirs_idx] + "/" + file_name; | |
7c673cae FG |
87 | std::unique_ptr<WritableFile> f; |
88 | env_->NewWritableFile(file_path, &f, EnvOptions()); | |
89 | std::string data(size, 'A'); | |
90 | EXPECT_OK(f->Append(data)); | |
91 | EXPECT_OK(f->Close()); | |
11fdf7f2 | 92 | sst_file_mgr_->OnAddFile(file_path, false); |
7c673cae FG |
93 | return file_path; |
94 | } | |
95 | ||
96 | void NewDeleteScheduler() { | |
11fdf7f2 TL |
97 | // Tests in this file are for DeleteScheduler component and dont create any |
98 | // DBs, so we need to set max_trash_db_ratio to 100% (instead of default | |
99 | // 25%) | |
100 | sst_file_mgr_.reset( | |
101 | new SstFileManagerImpl(env_, nullptr, rate_bytes_per_sec_, | |
102 | /* max_trash_db_ratio= */ 1.1, 128 * 1024)); | |
103 | delete_scheduler_ = sst_file_mgr_->delete_scheduler(); | |
7c673cae FG |
104 | } |
105 | ||
106 | Env* env_; | |
11fdf7f2 | 107 | std::vector<std::string> dummy_files_dirs_; |
7c673cae | 108 | int64_t rate_bytes_per_sec_; |
11fdf7f2 TL |
109 | DeleteScheduler* delete_scheduler_; |
110 | std::unique_ptr<SstFileManagerImpl> sst_file_mgr_; | |
7c673cae FG |
111 | }; |
112 | ||
113 | // Test the basic functionality of DeleteScheduler (Rate Limiting). | |
114 | // 1- Create 100 dummy files | |
115 | // 2- Delete the 100 dummy files using DeleteScheduler | |
116 | // --- Hold DeleteScheduler::BackgroundEmptyTrash --- | |
117 | // 3- Wait for DeleteScheduler to delete all files in trash | |
118 | // 4- Verify that BackgroundEmptyTrash used to correct penlties for the files | |
119 | // 5- Make sure that all created files were completely deleted | |
120 | TEST_F(DeleteSchedulerTest, BasicRateLimiting) { | |
121 | rocksdb::SyncPoint::GetInstance()->LoadDependency({ | |
122 | {"DeleteSchedulerTest::BasicRateLimiting:1", | |
123 | "DeleteScheduler::BackgroundEmptyTrash"}, | |
124 | }); | |
125 | ||
126 | std::vector<uint64_t> penalties; | |
127 | rocksdb::SyncPoint::GetInstance()->SetCallBack( | |
128 | "DeleteScheduler::BackgroundEmptyTrash:Wait", | |
129 | [&](void* arg) { penalties.push_back(*(static_cast<uint64_t*>(arg))); }); | |
11fdf7f2 TL |
130 | int dir_synced = 0; |
131 | rocksdb::SyncPoint::GetInstance()->SetCallBack( | |
132 | "DeleteScheduler::DeleteTrashFile::AfterSyncDir", [&](void* arg) { | |
133 | dir_synced++; | |
134 | std::string* dir = reinterpret_cast<std::string*>(arg); | |
135 | EXPECT_EQ(dummy_files_dirs_[0], *dir); | |
136 | }); | |
7c673cae FG |
137 | |
138 | int num_files = 100; // 100 files | |
139 | uint64_t file_size = 1024; // every file is 1 kb | |
140 | std::vector<uint64_t> delete_kbs_per_sec = {512, 200, 100, 50, 25}; | |
141 | ||
142 | for (size_t t = 0; t < delete_kbs_per_sec.size(); t++) { | |
143 | penalties.clear(); | |
144 | rocksdb::SyncPoint::GetInstance()->ClearTrace(); | |
145 | rocksdb::SyncPoint::GetInstance()->EnableProcessing(); | |
146 | ||
11fdf7f2 | 147 | DestroyAndCreateDir(dummy_files_dirs_[0]); |
7c673cae FG |
148 | rate_bytes_per_sec_ = delete_kbs_per_sec[t] * 1024; |
149 | NewDeleteScheduler(); | |
150 | ||
11fdf7f2 | 151 | dir_synced = 0; |
7c673cae FG |
152 | // Create 100 dummy files, every file is 1 Kb |
153 | std::vector<std::string> generated_files; | |
154 | for (int i = 0; i < num_files; i++) { | |
155 | std::string file_name = "file" + ToString(i) + ".data"; | |
156 | generated_files.push_back(NewDummyFile(file_name, file_size)); | |
157 | } | |
158 | ||
159 | // Delete dummy files and measure time spent to empty trash | |
160 | for (int i = 0; i < num_files; i++) { | |
11fdf7f2 TL |
161 | ASSERT_OK(delete_scheduler_->DeleteFile(generated_files[i], |
162 | dummy_files_dirs_[0])); | |
7c673cae | 163 | } |
11fdf7f2 | 164 | ASSERT_EQ(CountNormalFiles(), 0); |
7c673cae FG |
165 | |
166 | uint64_t delete_start_time = env_->NowMicros(); | |
167 | TEST_SYNC_POINT("DeleteSchedulerTest::BasicRateLimiting:1"); | |
168 | delete_scheduler_->WaitForEmptyTrash(); | |
169 | uint64_t time_spent_deleting = env_->NowMicros() - delete_start_time; | |
170 | ||
171 | auto bg_errors = delete_scheduler_->GetBackgroundErrors(); | |
172 | ASSERT_EQ(bg_errors.size(), 0); | |
173 | ||
174 | uint64_t total_files_size = 0; | |
175 | uint64_t expected_penlty = 0; | |
176 | ASSERT_EQ(penalties.size(), num_files); | |
177 | for (int i = 0; i < num_files; i++) { | |
178 | total_files_size += file_size; | |
179 | expected_penlty = ((total_files_size * 1000000) / rate_bytes_per_sec_); | |
180 | ASSERT_EQ(expected_penlty, penalties[i]); | |
181 | } | |
182 | ASSERT_GT(time_spent_deleting, expected_penlty * 0.9); | |
183 | ||
11fdf7f2 TL |
184 | ASSERT_EQ(num_files, dir_synced); |
185 | ||
186 | ASSERT_EQ(CountTrashFiles(), 0); | |
7c673cae FG |
187 | rocksdb::SyncPoint::GetInstance()->DisableProcessing(); |
188 | } | |
189 | } | |
190 | ||
11fdf7f2 TL |
191 | TEST_F(DeleteSchedulerTest, MultiDirectoryDeletionsScheduled) { |
192 | rocksdb::SyncPoint::GetInstance()->LoadDependency({ | |
193 | {"DeleteSchedulerTest::MultiDbPathDeletionsScheduled:1", | |
194 | "DeleteScheduler::BackgroundEmptyTrash"}, | |
195 | }); | |
196 | rocksdb::SyncPoint::GetInstance()->EnableProcessing(); | |
197 | rate_bytes_per_sec_ = 1 << 20; // 1MB | |
198 | NewDeleteScheduler(); | |
199 | ||
200 | // Generate dummy files in multiple directories | |
201 | const size_t kNumFiles = dummy_files_dirs_.size(); | |
202 | const size_t kFileSize = 1 << 10; // 1KB | |
203 | std::vector<std::string> generated_files; | |
204 | for (size_t i = 0; i < kNumFiles; i++) { | |
205 | generated_files.push_back(NewDummyFile("file", kFileSize, i)); | |
206 | ASSERT_EQ(1, CountNormalFiles(i)); | |
207 | } | |
208 | ||
209 | // Mark dummy files as trash | |
210 | for (size_t i = 0; i < kNumFiles; i++) { | |
211 | ASSERT_OK(delete_scheduler_->DeleteFile(generated_files[i], "")); | |
212 | ASSERT_EQ(0, CountNormalFiles(i)); | |
213 | ASSERT_EQ(1, CountTrashFiles(i)); | |
214 | } | |
215 | TEST_SYNC_POINT("DeleteSchedulerTest::MultiDbPathDeletionsScheduled:1"); | |
216 | delete_scheduler_->WaitForEmptyTrash(); | |
217 | ||
218 | // Verify dummy files eventually got deleted | |
219 | for (size_t i = 0; i < kNumFiles; i++) { | |
220 | ASSERT_EQ(0, CountNormalFiles(i)); | |
221 | ASSERT_EQ(0, CountTrashFiles(i)); | |
222 | } | |
223 | ||
224 | rocksdb::SyncPoint::GetInstance()->DisableProcessing(); | |
225 | } | |
226 | ||
7c673cae FG |
227 | // Same as the BasicRateLimiting test but delete files in multiple threads. |
228 | // 1- Create 100 dummy files | |
229 | // 2- Delete the 100 dummy files using DeleteScheduler using 10 threads | |
230 | // --- Hold DeleteScheduler::BackgroundEmptyTrash --- | |
231 | // 3- Wait for DeleteScheduler to delete all files in queue | |
232 | // 4- Verify that BackgroundEmptyTrash used to correct penlties for the files | |
233 | // 5- Make sure that all created files were completely deleted | |
234 | TEST_F(DeleteSchedulerTest, RateLimitingMultiThreaded) { | |
235 | rocksdb::SyncPoint::GetInstance()->LoadDependency({ | |
236 | {"DeleteSchedulerTest::RateLimitingMultiThreaded:1", | |
237 | "DeleteScheduler::BackgroundEmptyTrash"}, | |
238 | }); | |
239 | ||
240 | std::vector<uint64_t> penalties; | |
241 | rocksdb::SyncPoint::GetInstance()->SetCallBack( | |
242 | "DeleteScheduler::BackgroundEmptyTrash:Wait", | |
243 | [&](void* arg) { penalties.push_back(*(static_cast<uint64_t*>(arg))); }); | |
244 | ||
245 | int thread_cnt = 10; | |
246 | int num_files = 10; // 10 files per thread | |
247 | uint64_t file_size = 1024; // every file is 1 kb | |
248 | ||
249 | std::vector<uint64_t> delete_kbs_per_sec = {512, 200, 100, 50, 25}; | |
250 | for (size_t t = 0; t < delete_kbs_per_sec.size(); t++) { | |
251 | penalties.clear(); | |
252 | rocksdb::SyncPoint::GetInstance()->ClearTrace(); | |
253 | rocksdb::SyncPoint::GetInstance()->EnableProcessing(); | |
254 | ||
11fdf7f2 | 255 | DestroyAndCreateDir(dummy_files_dirs_[0]); |
7c673cae FG |
256 | rate_bytes_per_sec_ = delete_kbs_per_sec[t] * 1024; |
257 | NewDeleteScheduler(); | |
258 | ||
259 | // Create 100 dummy files, every file is 1 Kb | |
260 | std::vector<std::string> generated_files; | |
261 | for (int i = 0; i < num_files * thread_cnt; i++) { | |
262 | std::string file_name = "file" + ToString(i) + ".data"; | |
263 | generated_files.push_back(NewDummyFile(file_name, file_size)); | |
264 | } | |
265 | ||
266 | // Delete dummy files using 10 threads and measure time spent to empty trash | |
267 | std::atomic<int> thread_num(0); | |
268 | std::vector<port::Thread> threads; | |
269 | std::function<void()> delete_thread = [&]() { | |
270 | int idx = thread_num.fetch_add(1); | |
271 | int range_start = idx * num_files; | |
272 | int range_end = range_start + num_files; | |
273 | for (int j = range_start; j < range_end; j++) { | |
11fdf7f2 | 274 | ASSERT_OK(delete_scheduler_->DeleteFile(generated_files[j], "")); |
7c673cae FG |
275 | } |
276 | }; | |
277 | ||
278 | for (int i = 0; i < thread_cnt; i++) { | |
279 | threads.emplace_back(delete_thread); | |
280 | } | |
281 | ||
282 | for (size_t i = 0; i < threads.size(); i++) { | |
283 | threads[i].join(); | |
284 | } | |
285 | ||
286 | uint64_t delete_start_time = env_->NowMicros(); | |
287 | TEST_SYNC_POINT("DeleteSchedulerTest::RateLimitingMultiThreaded:1"); | |
288 | delete_scheduler_->WaitForEmptyTrash(); | |
289 | uint64_t time_spent_deleting = env_->NowMicros() - delete_start_time; | |
290 | ||
291 | auto bg_errors = delete_scheduler_->GetBackgroundErrors(); | |
292 | ASSERT_EQ(bg_errors.size(), 0); | |
293 | ||
294 | uint64_t total_files_size = 0; | |
295 | uint64_t expected_penlty = 0; | |
296 | ASSERT_EQ(penalties.size(), num_files * thread_cnt); | |
297 | for (int i = 0; i < num_files * thread_cnt; i++) { | |
298 | total_files_size += file_size; | |
299 | expected_penlty = ((total_files_size * 1000000) / rate_bytes_per_sec_); | |
300 | ASSERT_EQ(expected_penlty, penalties[i]); | |
301 | } | |
302 | ASSERT_GT(time_spent_deleting, expected_penlty * 0.9); | |
303 | ||
11fdf7f2 TL |
304 | ASSERT_EQ(CountNormalFiles(), 0); |
305 | ASSERT_EQ(CountTrashFiles(), 0); | |
7c673cae FG |
306 | rocksdb::SyncPoint::GetInstance()->DisableProcessing(); |
307 | } | |
308 | } | |
309 | ||
310 | // Disable rate limiting by setting rate_bytes_per_sec_ to 0 and make sure | |
311 | // that when DeleteScheduler delete a file it delete it immediately and dont | |
312 | // move it to trash | |
313 | TEST_F(DeleteSchedulerTest, DisableRateLimiting) { | |
314 | int bg_delete_file = 0; | |
315 | rocksdb::SyncPoint::GetInstance()->SetCallBack( | |
316 | "DeleteScheduler::DeleteTrashFile:DeleteFile", | |
11fdf7f2 | 317 | [&](void* /*arg*/) { bg_delete_file++; }); |
7c673cae FG |
318 | |
319 | rocksdb::SyncPoint::GetInstance()->EnableProcessing(); | |
320 | ||
321 | rate_bytes_per_sec_ = 0; | |
322 | NewDeleteScheduler(); | |
323 | ||
324 | for (int i = 0; i < 10; i++) { | |
325 | // Every file we delete will be deleted immediately | |
326 | std::string dummy_file = NewDummyFile("dummy.data"); | |
11fdf7f2 | 327 | ASSERT_OK(delete_scheduler_->DeleteFile(dummy_file, "")); |
7c673cae | 328 | ASSERT_TRUE(env_->FileExists(dummy_file).IsNotFound()); |
11fdf7f2 TL |
329 | ASSERT_EQ(CountNormalFiles(), 0); |
330 | ASSERT_EQ(CountTrashFiles(), 0); | |
7c673cae FG |
331 | } |
332 | ||
333 | ASSERT_EQ(bg_delete_file, 0); | |
334 | ||
335 | rocksdb::SyncPoint::GetInstance()->DisableProcessing(); | |
336 | } | |
337 | ||
338 | // Testing that moving files to trash with the same name is not a problem | |
339 | // 1- Create 10 files with the same name "conflict.data" | |
340 | // 2- Delete the 10 files using DeleteScheduler | |
341 | // 3- Make sure that trash directory contain 10 files ("conflict.data" x 10) | |
342 | // --- Hold DeleteScheduler::BackgroundEmptyTrash --- | |
343 | // 4- Make sure that files are deleted from trash | |
344 | TEST_F(DeleteSchedulerTest, ConflictNames) { | |
345 | rocksdb::SyncPoint::GetInstance()->LoadDependency({ | |
346 | {"DeleteSchedulerTest::ConflictNames:1", | |
347 | "DeleteScheduler::BackgroundEmptyTrash"}, | |
348 | }); | |
349 | rocksdb::SyncPoint::GetInstance()->EnableProcessing(); | |
350 | ||
351 | rate_bytes_per_sec_ = 1024 * 1024; // 1 Mb/sec | |
352 | NewDeleteScheduler(); | |
353 | ||
354 | // Create "conflict.data" and move it to trash 10 times | |
355 | for (int i = 0; i < 10; i++) { | |
356 | std::string dummy_file = NewDummyFile("conflict.data"); | |
11fdf7f2 | 357 | ASSERT_OK(delete_scheduler_->DeleteFile(dummy_file, "")); |
7c673cae | 358 | } |
11fdf7f2 | 359 | ASSERT_EQ(CountNormalFiles(), 0); |
7c673cae | 360 | // 10 files ("conflict.data" x 10) in trash |
11fdf7f2 | 361 | ASSERT_EQ(CountTrashFiles(), 10); |
7c673cae FG |
362 | |
363 | // Hold BackgroundEmptyTrash | |
364 | TEST_SYNC_POINT("DeleteSchedulerTest::ConflictNames:1"); | |
365 | delete_scheduler_->WaitForEmptyTrash(); | |
11fdf7f2 | 366 | ASSERT_EQ(CountTrashFiles(), 0); |
7c673cae FG |
367 | |
368 | auto bg_errors = delete_scheduler_->GetBackgroundErrors(); | |
369 | ASSERT_EQ(bg_errors.size(), 0); | |
370 | ||
371 | rocksdb::SyncPoint::GetInstance()->DisableProcessing(); | |
372 | } | |
373 | ||
374 | // 1- Create 10 dummy files | |
375 | // 2- Delete the 10 files using DeleteScheduler (move them to trsah) | |
376 | // 3- Delete the 10 files directly (using env_->DeleteFile) | |
377 | // --- Hold DeleteScheduler::BackgroundEmptyTrash --- | |
378 | // 4- Make sure that DeleteScheduler failed to delete the 10 files and | |
379 | // reported 10 background errors | |
380 | TEST_F(DeleteSchedulerTest, BackgroundError) { | |
381 | rocksdb::SyncPoint::GetInstance()->LoadDependency({ | |
382 | {"DeleteSchedulerTest::BackgroundError:1", | |
383 | "DeleteScheduler::BackgroundEmptyTrash"}, | |
384 | }); | |
385 | rocksdb::SyncPoint::GetInstance()->EnableProcessing(); | |
386 | ||
387 | rate_bytes_per_sec_ = 1024 * 1024; // 1 Mb/sec | |
388 | NewDeleteScheduler(); | |
389 | ||
390 | // Generate 10 dummy files and move them to trash | |
391 | for (int i = 0; i < 10; i++) { | |
392 | std::string file_name = "data_" + ToString(i) + ".data"; | |
11fdf7f2 | 393 | ASSERT_OK(delete_scheduler_->DeleteFile(NewDummyFile(file_name), "")); |
7c673cae | 394 | } |
11fdf7f2 TL |
395 | ASSERT_EQ(CountNormalFiles(), 0); |
396 | ASSERT_EQ(CountTrashFiles(), 10); | |
7c673cae FG |
397 | |
398 | // Delete 10 files from trash, this will cause background errors in | |
399 | // BackgroundEmptyTrash since we already deleted the files it was | |
400 | // goind to delete | |
401 | for (int i = 0; i < 10; i++) { | |
11fdf7f2 TL |
402 | std::string file_name = "data_" + ToString(i) + ".data.trash"; |
403 | ASSERT_OK(env_->DeleteFile(dummy_files_dirs_[0] + "/" + file_name)); | |
7c673cae FG |
404 | } |
405 | ||
406 | // Hold BackgroundEmptyTrash | |
407 | TEST_SYNC_POINT("DeleteSchedulerTest::BackgroundError:1"); | |
408 | delete_scheduler_->WaitForEmptyTrash(); | |
409 | auto bg_errors = delete_scheduler_->GetBackgroundErrors(); | |
410 | ASSERT_EQ(bg_errors.size(), 10); | |
411 | ||
412 | rocksdb::SyncPoint::GetInstance()->DisableProcessing(); | |
413 | } | |
414 | ||
415 | // 1- Create 10 dummy files | |
416 | // 2- Delete 10 dummy files using DeleteScheduler | |
417 | // 3- Wait for DeleteScheduler to delete all files in queue | |
418 | // 4- Make sure all files in trash directory were deleted | |
419 | // 5- Repeat previous steps 5 times | |
420 | TEST_F(DeleteSchedulerTest, StartBGEmptyTrashMultipleTimes) { | |
421 | int bg_delete_file = 0; | |
422 | rocksdb::SyncPoint::GetInstance()->SetCallBack( | |
423 | "DeleteScheduler::DeleteTrashFile:DeleteFile", | |
11fdf7f2 | 424 | [&](void* /*arg*/) { bg_delete_file++; }); |
7c673cae FG |
425 | rocksdb::SyncPoint::GetInstance()->EnableProcessing(); |
426 | ||
427 | rate_bytes_per_sec_ = 1024 * 1024; // 1 MB / sec | |
428 | NewDeleteScheduler(); | |
429 | ||
430 | // Move files to trash, wait for empty trash, start again | |
431 | for (int run = 1; run <= 5; run++) { | |
432 | // Generate 10 dummy files and move them to trash | |
433 | for (int i = 0; i < 10; i++) { | |
434 | std::string file_name = "data_" + ToString(i) + ".data"; | |
11fdf7f2 | 435 | ASSERT_OK(delete_scheduler_->DeleteFile(NewDummyFile(file_name), "")); |
7c673cae | 436 | } |
11fdf7f2 | 437 | ASSERT_EQ(CountNormalFiles(), 0); |
7c673cae FG |
438 | delete_scheduler_->WaitForEmptyTrash(); |
439 | ASSERT_EQ(bg_delete_file, 10 * run); | |
11fdf7f2 | 440 | ASSERT_EQ(CountTrashFiles(), 0); |
7c673cae FG |
441 | |
442 | auto bg_errors = delete_scheduler_->GetBackgroundErrors(); | |
443 | ASSERT_EQ(bg_errors.size(), 0); | |
444 | } | |
445 | ||
446 | ASSERT_EQ(bg_delete_file, 50); | |
447 | rocksdb::SyncPoint::GetInstance()->EnableProcessing(); | |
448 | } | |
449 | ||
11fdf7f2 | 450 | TEST_F(DeleteSchedulerTest, DeletePartialFile) { |
7c673cae | 451 | int bg_delete_file = 0; |
11fdf7f2 | 452 | int bg_fsync = 0; |
7c673cae FG |
453 | rocksdb::SyncPoint::GetInstance()->SetCallBack( |
454 | "DeleteScheduler::DeleteTrashFile:DeleteFile", | |
11fdf7f2 TL |
455 | [&](void*) { bg_delete_file++; }); |
456 | rocksdb::SyncPoint::GetInstance()->SetCallBack( | |
457 | "DeleteScheduler::DeleteTrashFile:Fsync", [&](void*) { bg_fsync++; }); | |
7c673cae FG |
458 | rocksdb::SyncPoint::GetInstance()->EnableProcessing(); |
459 | ||
11fdf7f2 | 460 | rate_bytes_per_sec_ = 1024 * 1024; // 1 MB / sec |
7c673cae FG |
461 | NewDeleteScheduler(); |
462 | ||
11fdf7f2 TL |
463 | // Should delete in 4 batch |
464 | ASSERT_OK( | |
465 | delete_scheduler_->DeleteFile(NewDummyFile("data_1", 500 * 1024), "")); | |
466 | ASSERT_OK( | |
467 | delete_scheduler_->DeleteFile(NewDummyFile("data_2", 100 * 1024), "")); | |
468 | // Should delete in 2 batch | |
469 | ASSERT_OK( | |
470 | delete_scheduler_->DeleteFile(NewDummyFile("data_2", 200 * 1024), "")); | |
7c673cae | 471 | |
11fdf7f2 | 472 | delete_scheduler_->WaitForEmptyTrash(); |
7c673cae | 473 | |
11fdf7f2 TL |
474 | auto bg_errors = delete_scheduler_->GetBackgroundErrors(); |
475 | ASSERT_EQ(bg_errors.size(), 0); | |
476 | ASSERT_EQ(7, bg_delete_file); | |
477 | ASSERT_EQ(4, bg_fsync); | |
478 | rocksdb::SyncPoint::GetInstance()->EnableProcessing(); | |
479 | } | |
7c673cae | 480 | |
11fdf7f2 TL |
481 | #ifdef OS_LINUX |
482 | TEST_F(DeleteSchedulerTest, NoPartialDeleteWithLink) { | |
483 | int bg_delete_file = 0; | |
484 | int bg_fsync = 0; | |
485 | rocksdb::SyncPoint::GetInstance()->SetCallBack( | |
486 | "DeleteScheduler::DeleteTrashFile:DeleteFile", | |
487 | [&](void*) { bg_delete_file++; }); | |
488 | rocksdb::SyncPoint::GetInstance()->SetCallBack( | |
489 | "DeleteScheduler::DeleteTrashFile:Fsync", [&](void*) { bg_fsync++; }); | |
490 | rocksdb::SyncPoint::GetInstance()->EnableProcessing(); | |
491 | ||
492 | rate_bytes_per_sec_ = 1024 * 1024; // 1 MB / sec | |
493 | NewDeleteScheduler(); | |
494 | ||
495 | std::string file1 = NewDummyFile("data_1", 500 * 1024); | |
496 | std::string file2 = NewDummyFile("data_2", 100 * 1024); | |
497 | ||
498 | ASSERT_OK(env_->LinkFile(file1, dummy_files_dirs_[0] + "/data_1b")); | |
499 | ASSERT_OK(env_->LinkFile(file2, dummy_files_dirs_[0] + "/data_2b")); | |
500 | ||
501 | // Should delete in 4 batch if there is no hardlink | |
502 | ASSERT_OK(delete_scheduler_->DeleteFile(file1, "")); | |
503 | ASSERT_OK(delete_scheduler_->DeleteFile(file2, "")); | |
504 | ||
505 | delete_scheduler_->WaitForEmptyTrash(); | |
506 | ||
507 | auto bg_errors = delete_scheduler_->GetBackgroundErrors(); | |
508 | ASSERT_EQ(bg_errors.size(), 0); | |
509 | ASSERT_EQ(2, bg_delete_file); | |
510 | ASSERT_EQ(0, bg_fsync); | |
511 | rocksdb::SyncPoint::GetInstance()->EnableProcessing(); | |
7c673cae | 512 | } |
11fdf7f2 | 513 | #endif |
7c673cae | 514 | |
11fdf7f2 TL |
515 | // 1- Create a DeleteScheduler with very slow rate limit (1 Byte / sec) |
516 | // 2- Delete 100 files using DeleteScheduler | |
517 | // 3- Delete the DeleteScheduler (call the destructor while queue is not empty) | |
518 | // 4- Make sure that not all files were deleted from trash and that | |
519 | // DeleteScheduler background thread did not delete all files | |
520 | TEST_F(DeleteSchedulerTest, DestructorWithNonEmptyQueue) { | |
7c673cae FG |
521 | int bg_delete_file = 0; |
522 | rocksdb::SyncPoint::GetInstance()->SetCallBack( | |
523 | "DeleteScheduler::DeleteTrashFile:DeleteFile", | |
11fdf7f2 | 524 | [&](void* /*arg*/) { bg_delete_file++; }); |
7c673cae FG |
525 | rocksdb::SyncPoint::GetInstance()->EnableProcessing(); |
526 | ||
11fdf7f2 | 527 | rate_bytes_per_sec_ = 1; // 1 Byte / sec |
7c673cae FG |
528 | NewDeleteScheduler(); |
529 | ||
11fdf7f2 | 530 | for (int i = 0; i < 100; i++) { |
7c673cae | 531 | std::string file_name = "data_" + ToString(i) + ".data"; |
11fdf7f2 | 532 | ASSERT_OK(delete_scheduler_->DeleteFile(NewDummyFile(file_name), "")); |
7c673cae FG |
533 | } |
534 | ||
11fdf7f2 TL |
535 | // Deleting 100 files will need >28 hours to delete |
536 | // we will delete the DeleteScheduler while delete queue is not empty | |
537 | sst_file_mgr_.reset(); | |
538 | ||
539 | ASSERT_LT(bg_delete_file, 100); | |
540 | ASSERT_GT(CountTrashFiles(), 0); | |
7c673cae FG |
541 | |
542 | rocksdb::SyncPoint::GetInstance()->DisableProcessing(); | |
543 | } | |
544 | ||
545 | TEST_F(DeleteSchedulerTest, DISABLED_DynamicRateLimiting1) { | |
546 | std::vector<uint64_t> penalties; | |
547 | int bg_delete_file = 0; | |
548 | int fg_delete_file = 0; | |
549 | rocksdb::SyncPoint::GetInstance()->SetCallBack( | |
550 | "DeleteScheduler::DeleteTrashFile:DeleteFile", | |
11fdf7f2 | 551 | [&](void* /*arg*/) { bg_delete_file++; }); |
7c673cae FG |
552 | rocksdb::SyncPoint::GetInstance()->SetCallBack( |
553 | "DeleteScheduler::DeleteFile", | |
11fdf7f2 | 554 | [&](void* /*arg*/) { fg_delete_file++; }); |
7c673cae FG |
555 | rocksdb::SyncPoint::GetInstance()->SetCallBack( |
556 | "DeleteScheduler::BackgroundEmptyTrash:Wait", | |
557 | [&](void* arg) { penalties.push_back(*(static_cast<int*>(arg))); }); | |
558 | ||
559 | rocksdb::SyncPoint::GetInstance()->LoadDependency({ | |
560 | {"DeleteSchedulerTest::DynamicRateLimiting1:1", | |
561 | "DeleteScheduler::BackgroundEmptyTrash"}, | |
562 | }); | |
563 | rocksdb::SyncPoint::GetInstance()->EnableProcessing(); | |
564 | ||
565 | rate_bytes_per_sec_ = 0; // Disable rate limiting initially | |
566 | NewDeleteScheduler(); | |
567 | ||
568 | ||
569 | int num_files = 10; // 10 files | |
570 | uint64_t file_size = 1024; // every file is 1 kb | |
571 | ||
572 | std::vector<int64_t> delete_kbs_per_sec = {512, 200, 0, 100, 50, -2, 25}; | |
573 | for (size_t t = 0; t < delete_kbs_per_sec.size(); t++) { | |
574 | penalties.clear(); | |
575 | bg_delete_file = 0; | |
576 | fg_delete_file = 0; | |
577 | rocksdb::SyncPoint::GetInstance()->ClearTrace(); | |
578 | rocksdb::SyncPoint::GetInstance()->EnableProcessing(); | |
579 | ||
11fdf7f2 | 580 | DestroyAndCreateDir(dummy_files_dirs_[0]); |
7c673cae FG |
581 | rate_bytes_per_sec_ = delete_kbs_per_sec[t] * 1024; |
582 | delete_scheduler_->SetRateBytesPerSecond(rate_bytes_per_sec_); | |
583 | ||
584 | // Create 100 dummy files, every file is 1 Kb | |
585 | std::vector<std::string> generated_files; | |
586 | for (int i = 0; i < num_files; i++) { | |
587 | std::string file_name = "file" + ToString(i) + ".data"; | |
588 | generated_files.push_back(NewDummyFile(file_name, file_size)); | |
589 | } | |
590 | ||
591 | // Delete dummy files and measure time spent to empty trash | |
592 | for (int i = 0; i < num_files; i++) { | |
11fdf7f2 | 593 | ASSERT_OK(delete_scheduler_->DeleteFile(generated_files[i], "")); |
7c673cae | 594 | } |
11fdf7f2 | 595 | ASSERT_EQ(CountNormalFiles(), 0); |
7c673cae FG |
596 | |
597 | if (rate_bytes_per_sec_ > 0) { | |
598 | uint64_t delete_start_time = env_->NowMicros(); | |
599 | TEST_SYNC_POINT("DeleteSchedulerTest::DynamicRateLimiting1:1"); | |
600 | delete_scheduler_->WaitForEmptyTrash(); | |
601 | uint64_t time_spent_deleting = env_->NowMicros() - delete_start_time; | |
602 | ||
603 | auto bg_errors = delete_scheduler_->GetBackgroundErrors(); | |
604 | ASSERT_EQ(bg_errors.size(), 0); | |
605 | ||
606 | uint64_t total_files_size = 0; | |
607 | uint64_t expected_penlty = 0; | |
608 | ASSERT_EQ(penalties.size(), num_files); | |
609 | for (int i = 0; i < num_files; i++) { | |
610 | total_files_size += file_size; | |
611 | expected_penlty = ((total_files_size * 1000000) / rate_bytes_per_sec_); | |
612 | ASSERT_EQ(expected_penlty, penalties[i]); | |
613 | } | |
614 | ASSERT_GT(time_spent_deleting, expected_penlty * 0.9); | |
615 | ASSERT_EQ(bg_delete_file, num_files); | |
616 | ASSERT_EQ(fg_delete_file, 0); | |
617 | } else { | |
618 | ASSERT_EQ(penalties.size(), 0); | |
619 | ASSERT_EQ(bg_delete_file, 0); | |
620 | ASSERT_EQ(fg_delete_file, num_files); | |
621 | } | |
622 | ||
11fdf7f2 | 623 | ASSERT_EQ(CountTrashFiles(), 0); |
7c673cae FG |
624 | rocksdb::SyncPoint::GetInstance()->DisableProcessing(); |
625 | } | |
626 | } | |
627 | ||
11fdf7f2 TL |
628 | TEST_F(DeleteSchedulerTest, ImmediateDeleteOn25PercDBSize) { |
629 | int bg_delete_file = 0; | |
630 | int fg_delete_file = 0; | |
631 | rocksdb::SyncPoint::GetInstance()->SetCallBack( | |
632 | "DeleteScheduler::DeleteTrashFile:DeleteFile", | |
633 | [&](void* /*arg*/) { bg_delete_file++; }); | |
634 | rocksdb::SyncPoint::GetInstance()->SetCallBack( | |
635 | "DeleteScheduler::DeleteFile", [&](void* /*arg*/) { fg_delete_file++; }); | |
636 | ||
637 | rocksdb::SyncPoint::GetInstance()->EnableProcessing(); | |
638 | ||
639 | int num_files = 100; // 100 files | |
640 | uint64_t file_size = 1024 * 10; // 100 KB as a file size | |
641 | rate_bytes_per_sec_ = 1; // 1 byte per sec (very slow trash delete) | |
642 | ||
643 | NewDeleteScheduler(); | |
644 | delete_scheduler_->SetMaxTrashDBRatio(0.25); | |
645 | ||
646 | std::vector<std::string> generated_files; | |
647 | for (int i = 0; i < num_files; i++) { | |
648 | std::string file_name = "file" + ToString(i) + ".data"; | |
649 | generated_files.push_back(NewDummyFile(file_name, file_size)); | |
650 | } | |
651 | ||
652 | for (std::string& file_name : generated_files) { | |
653 | delete_scheduler_->DeleteFile(file_name, ""); | |
654 | } | |
655 | ||
656 | // When we end up with 26 files in trash we will start | |
657 | // deleting new files immediately | |
658 | ASSERT_EQ(fg_delete_file, 74); | |
659 | ||
660 | rocksdb::SyncPoint::GetInstance()->DisableProcessing(); | |
661 | } | |
662 | ||
663 | TEST_F(DeleteSchedulerTest, IsTrashCheck) { | |
664 | // Trash files | |
665 | ASSERT_TRUE(DeleteScheduler::IsTrashFile("x.trash")); | |
666 | ASSERT_TRUE(DeleteScheduler::IsTrashFile(".trash")); | |
667 | ASSERT_TRUE(DeleteScheduler::IsTrashFile("abc.sst.trash")); | |
668 | ASSERT_TRUE(DeleteScheduler::IsTrashFile("/a/b/c/abc..sst.trash")); | |
669 | ASSERT_TRUE(DeleteScheduler::IsTrashFile("log.trash")); | |
670 | ASSERT_TRUE(DeleteScheduler::IsTrashFile("^^^^^.log.trash")); | |
671 | ASSERT_TRUE(DeleteScheduler::IsTrashFile("abc.t.trash")); | |
672 | ||
673 | // Not trash files | |
674 | ASSERT_FALSE(DeleteScheduler::IsTrashFile("abc.sst")); | |
675 | ASSERT_FALSE(DeleteScheduler::IsTrashFile("abc.txt")); | |
676 | ASSERT_FALSE(DeleteScheduler::IsTrashFile("/a/b/c/abc.sst")); | |
677 | ASSERT_FALSE(DeleteScheduler::IsTrashFile("/a/b/c/abc.sstrash")); | |
678 | ASSERT_FALSE(DeleteScheduler::IsTrashFile("^^^^^.trashh")); | |
679 | ASSERT_FALSE(DeleteScheduler::IsTrashFile("abc.ttrash")); | |
680 | ASSERT_FALSE(DeleteScheduler::IsTrashFile(".ttrash")); | |
681 | ASSERT_FALSE(DeleteScheduler::IsTrashFile("abc.trashx")); | |
682 | } | |
683 | ||
7c673cae FG |
684 | } // namespace rocksdb |
685 | ||
686 | int main(int argc, char** argv) { | |
687 | ::testing::InitGoogleTest(&argc, argv); | |
688 | return RUN_ALL_TESTS(); | |
689 | } | |
690 | ||
691 | #else | |
11fdf7f2 | 692 | int main(int /*argc*/, char** /*argv*/) { |
7c673cae FG |
693 | printf("DeleteScheduler is not supported in ROCKSDB_LITE\n"); |
694 | return 0; | |
695 | } | |
696 | #endif // ROCKSDB_LITE |