]>
Commit | Line | Data |
---|---|---|
7c673cae | 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. |
11fdf7f2 TL |
2 | // This source code is licensed under both the GPLv2 (found in the |
3 | // COPYING file in the root directory) and Apache 2.0 License | |
4 | // (found in the LICENSE.Apache file in the root directory). | |
7c673cae FG |
5 | |
6 | #pragma once | |
7 | ||
8 | #ifndef ROCKSDB_LITE | |
9 | ||
10 | #include <map> | |
11 | #include <queue> | |
12 | #include <string> | |
13 | #include <thread> | |
14 | ||
15 | #include "monitoring/instrumented_mutex.h" | |
16 | #include "port/port.h" | |
17 | ||
18 | #include "rocksdb/status.h" | |
19 | ||
20 | namespace rocksdb { | |
21 | ||
22 | class Env; | |
23 | class Logger; | |
24 | class SstFileManagerImpl; | |
25 | ||
26 | // DeleteScheduler allows the DB to enforce a rate limit on file deletion, | |
11fdf7f2 | 27 | // Instead of deleteing files immediately, files are marked as trash |
7c673cae FG |
28 | // and deleted in a background thread that apply sleep penlty between deletes |
29 | // if they are happening in a rate faster than rate_bytes_per_sec, | |
30 | // | |
31 | // Rate limiting can be turned off by setting rate_bytes_per_sec = 0, In this | |
32 | // case DeleteScheduler will delete files immediately. | |
33 | class DeleteScheduler { | |
34 | public: | |
11fdf7f2 TL |
35 | DeleteScheduler(Env* env, int64_t rate_bytes_per_sec, Logger* info_log, |
36 | SstFileManagerImpl* sst_file_manager, | |
37 | double max_trash_db_ratio, uint64_t bytes_max_delete_chunk); | |
7c673cae FG |
38 | |
39 | ~DeleteScheduler(); | |
40 | ||
41 | // Return delete rate limit in bytes per second | |
42 | int64_t GetRateBytesPerSecond() { return rate_bytes_per_sec_.load(); } | |
43 | ||
44 | // Set delete rate limit in bytes per second | |
45 | void SetRateBytesPerSecond(int64_t bytes_per_sec) { | |
11fdf7f2 | 46 | rate_bytes_per_sec_.store(bytes_per_sec); |
7c673cae FG |
47 | } |
48 | ||
494da23a TL |
49 | // Mark file as trash directory and schedule it's deletion. If force_bg is |
50 | // set, it forces the file to always be deleted in the background thread, | |
51 | // except when rate limiting is disabled | |
52 | Status DeleteFile(const std::string& fname, const std::string& dir_to_sync, | |
53 | const bool force_bg = false); | |
7c673cae FG |
54 | |
55 | // Wait for all files being deleteing in the background to finish or for | |
56 | // destructor to be called. | |
57 | void WaitForEmptyTrash(); | |
58 | ||
59 | // Return a map containing errors that happened in BackgroundEmptyTrash | |
60 | // file_path => error status | |
61 | std::map<std::string, Status> GetBackgroundErrors(); | |
62 | ||
11fdf7f2 TL |
63 | uint64_t GetTotalTrashSize() { return total_trash_size_.load(); } |
64 | ||
65 | // Return trash/DB size ratio where new files will be deleted immediately | |
66 | double GetMaxTrashDBRatio() { | |
67 | return max_trash_db_ratio_.load(); | |
68 | } | |
69 | ||
70 | // Update trash/DB size ratio where new files will be deleted immediately | |
71 | void SetMaxTrashDBRatio(double r) { | |
72 | assert(r >= 0); | |
73 | max_trash_db_ratio_.store(r); | |
74 | } | |
75 | ||
76 | static const std::string kTrashExtension; | |
77 | static bool IsTrashFile(const std::string& file_path); | |
78 | ||
79 | // Check if there are any .trash filse in path, and schedule their deletion | |
80 | // Or delete immediately if sst_file_manager is nullptr | |
81 | static Status CleanupDirectory(Env* env, SstFileManagerImpl* sfm, | |
82 | const std::string& path); | |
83 | ||
7c673cae | 84 | private: |
11fdf7f2 | 85 | Status MarkAsTrash(const std::string& file_path, std::string* path_in_trash); |
7c673cae FG |
86 | |
87 | Status DeleteTrashFile(const std::string& path_in_trash, | |
11fdf7f2 TL |
88 | const std::string& dir_to_sync, |
89 | uint64_t* deleted_bytes, bool* is_complete); | |
7c673cae FG |
90 | |
91 | void BackgroundEmptyTrash(); | |
92 | ||
93 | Env* env_; | |
11fdf7f2 TL |
94 | // total size of trash files |
95 | std::atomic<uint64_t> total_trash_size_; | |
7c673cae FG |
96 | // Maximum number of bytes that should be deleted per second |
97 | std::atomic<int64_t> rate_bytes_per_sec_; | |
98 | // Mutex to protect queue_, pending_files_, bg_errors_, closing_ | |
99 | InstrumentedMutex mu_; | |
11fdf7f2 TL |
100 | |
101 | struct FileAndDir { | |
102 | FileAndDir(const std::string& f, const std::string& d) : fname(f), dir(d) {} | |
103 | std::string fname; | |
104 | std::string dir; // empty will be skipped. | |
105 | }; | |
106 | ||
107 | // Queue of trash files that need to be deleted | |
108 | std::queue<FileAndDir> queue_; | |
109 | // Number of trash files that are waiting to be deleted | |
7c673cae | 110 | int32_t pending_files_; |
11fdf7f2 | 111 | uint64_t bytes_max_delete_chunk_; |
7c673cae FG |
112 | // Errors that happened in BackgroundEmptyTrash (file_path => error) |
113 | std::map<std::string, Status> bg_errors_; | |
11fdf7f2 TL |
114 | |
115 | bool num_link_error_printed_ = false; | |
7c673cae FG |
116 | // Set to true in ~DeleteScheduler() to force BackgroundEmptyTrash to stop |
117 | bool closing_; | |
118 | // Condition variable signaled in these conditions | |
119 | // - pending_files_ value change from 0 => 1 | |
120 | // - pending_files_ value change from 1 => 0 | |
121 | // - closing_ value is set to true | |
122 | InstrumentedCondVar cv_; | |
123 | // Background thread running BackgroundEmptyTrash | |
124 | std::unique_ptr<port::Thread> bg_thread_; | |
125 | // Mutex to protect threads from file name conflicts | |
126 | InstrumentedMutex file_move_mu_; | |
127 | Logger* info_log_; | |
128 | SstFileManagerImpl* sst_file_manager_; | |
11fdf7f2 TL |
129 | // If the trash size constitutes for more than this fraction of the total DB |
130 | // size we will start deleting new files passed to DeleteScheduler | |
131 | // immediately | |
132 | std::atomic<double> max_trash_db_ratio_; | |
7c673cae FG |
133 | static const uint64_t kMicrosInSecond = 1000 * 1000LL; |
134 | }; | |
135 | ||
136 | } // namespace rocksdb | |
137 | ||
138 | #endif // ROCKSDB_LITE |