]>
Commit | Line | Data |
---|---|---|
494da23a TL |
1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. |
2 | // This source code is licensed under both the GPLv2 (found in the | |
3 | // COPYING file in the root directory) and Apache 2.0 License | |
4 | // (found in the LICENSE.Apache file in the root directory). | |
5 | // | |
6 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. | |
7 | // Use of this source code is governed by a BSD-style license that can be | |
8 | // found in the LICENSE file. See the AUTHORS file for names of contributors. | |
9 | ||
f67539c2 | 10 | #include "db/compaction/compaction_picker_fifo.h" |
494da23a TL |
11 | #ifndef ROCKSDB_LITE |
12 | ||
f67539c2 | 13 | #include <cinttypes> |
494da23a TL |
14 | #include <string> |
15 | #include <vector> | |
16 | #include "db/column_family.h" | |
f67539c2 | 17 | #include "logging/log_buffer.h" |
494da23a TL |
18 | #include "util/string_util.h" |
19 | ||
f67539c2 | 20 | namespace ROCKSDB_NAMESPACE { |
494da23a TL |
21 | namespace { |
22 | uint64_t GetTotalFilesSize(const std::vector<FileMetaData*>& files) { | |
23 | uint64_t total_size = 0; | |
24 | for (const auto& f : files) { | |
25 | total_size += f->fd.file_size; | |
26 | } | |
27 | return total_size; | |
28 | } | |
29 | } // anonymous namespace | |
30 | ||
31 | bool FIFOCompactionPicker::NeedsCompaction( | |
32 | const VersionStorageInfo* vstorage) const { | |
33 | const int kLevel0 = 0; | |
34 | return vstorage->CompactionScore(kLevel0) >= 1; | |
35 | } | |
36 | ||
37 | Compaction* FIFOCompactionPicker::PickTTLCompaction( | |
38 | const std::string& cf_name, const MutableCFOptions& mutable_cf_options, | |
39 | VersionStorageInfo* vstorage, LogBuffer* log_buffer) { | |
40 | assert(mutable_cf_options.ttl > 0); | |
41 | ||
42 | const int kLevel0 = 0; | |
43 | const std::vector<FileMetaData*>& level_files = vstorage->LevelFiles(kLevel0); | |
44 | uint64_t total_size = GetTotalFilesSize(level_files); | |
45 | ||
46 | int64_t _current_time; | |
47 | auto status = ioptions_.env->GetCurrentTime(&_current_time); | |
48 | if (!status.ok()) { | |
49 | ROCKS_LOG_BUFFER(log_buffer, | |
50 | "[%s] FIFO compaction: Couldn't get current time: %s. " | |
51 | "Not doing compactions based on TTL. ", | |
52 | cf_name.c_str(), status.ToString().c_str()); | |
53 | return nullptr; | |
54 | } | |
55 | const uint64_t current_time = static_cast<uint64_t>(_current_time); | |
56 | ||
f67539c2 TL |
57 | if (!level0_compactions_in_progress_.empty()) { |
58 | ROCKS_LOG_BUFFER( | |
59 | log_buffer, | |
60 | "[%s] FIFO compaction: Already executing compaction. No need " | |
61 | "to run parallel compactions since compactions are very fast", | |
62 | cf_name.c_str()); | |
63 | return nullptr; | |
64 | } | |
65 | ||
494da23a TL |
66 | std::vector<CompactionInputFiles> inputs; |
67 | inputs.emplace_back(); | |
68 | inputs[0].level = 0; | |
69 | ||
70 | // avoid underflow | |
71 | if (current_time > mutable_cf_options.ttl) { | |
72 | for (auto ritr = level_files.rbegin(); ritr != level_files.rend(); ++ritr) { | |
f67539c2 TL |
73 | FileMetaData* f = *ritr; |
74 | if (f->fd.table_reader && f->fd.table_reader->GetTableProperties()) { | |
75 | uint64_t creation_time = | |
494da23a TL |
76 | f->fd.table_reader->GetTableProperties()->creation_time; |
77 | if (creation_time == 0 || | |
78 | creation_time >= (current_time - mutable_cf_options.ttl)) { | |
79 | break; | |
80 | } | |
81 | total_size -= f->compensated_file_size; | |
82 | inputs[0].files.push_back(f); | |
83 | } | |
84 | } | |
85 | } | |
86 | ||
87 | // Return a nullptr and proceed to size-based FIFO compaction if: | |
88 | // 1. there are no files older than ttl OR | |
89 | // 2. there are a few files older than ttl, but deleting them will not bring | |
90 | // the total size to be less than max_table_files_size threshold. | |
91 | if (inputs[0].files.empty() || | |
92 | total_size > | |
93 | mutable_cf_options.compaction_options_fifo.max_table_files_size) { | |
94 | return nullptr; | |
95 | } | |
96 | ||
97 | for (const auto& f : inputs[0].files) { | |
f67539c2 TL |
98 | uint64_t creation_time = 0; |
99 | if (f && f->fd.table_reader && f->fd.table_reader->GetTableProperties()) { | |
100 | creation_time = f->fd.table_reader->GetTableProperties()->creation_time; | |
101 | } | |
494da23a TL |
102 | ROCKS_LOG_BUFFER(log_buffer, |
103 | "[%s] FIFO compaction: picking file %" PRIu64 | |
104 | " with creation time %" PRIu64 " for deletion", | |
f67539c2 | 105 | cf_name.c_str(), f->fd.GetNumber(), creation_time); |
494da23a TL |
106 | } |
107 | ||
108 | Compaction* c = new Compaction( | |
109 | vstorage, ioptions_, mutable_cf_options, std::move(inputs), 0, 0, 0, 0, | |
110 | kNoCompression, ioptions_.compression_opts, /* max_subcompactions */ 0, | |
111 | {}, /* is manual */ false, vstorage->CompactionScore(0), | |
112 | /* is deletion compaction */ true, CompactionReason::kFIFOTtl); | |
113 | return c; | |
114 | } | |
115 | ||
116 | Compaction* FIFOCompactionPicker::PickSizeCompaction( | |
117 | const std::string& cf_name, const MutableCFOptions& mutable_cf_options, | |
118 | VersionStorageInfo* vstorage, LogBuffer* log_buffer) { | |
119 | const int kLevel0 = 0; | |
120 | const std::vector<FileMetaData*>& level_files = vstorage->LevelFiles(kLevel0); | |
121 | uint64_t total_size = GetTotalFilesSize(level_files); | |
122 | ||
123 | if (total_size <= | |
124 | mutable_cf_options.compaction_options_fifo.max_table_files_size || | |
125 | level_files.size() == 0) { | |
126 | // total size not exceeded | |
127 | if (mutable_cf_options.compaction_options_fifo.allow_compaction && | |
128 | level_files.size() > 0) { | |
129 | CompactionInputFiles comp_inputs; | |
130 | // try to prevent same files from being compacted multiple times, which | |
131 | // could produce large files that may never TTL-expire. Achieve this by | |
132 | // disallowing compactions with files larger than memtable (inflate its | |
133 | // size by 10% to account for uncompressed L0 files that may have size | |
134 | // slightly greater than memtable size limit). | |
135 | size_t max_compact_bytes_per_del_file = | |
136 | static_cast<size_t>(MultiplyCheckOverflow( | |
137 | static_cast<uint64_t>(mutable_cf_options.write_buffer_size), | |
138 | 1.1)); | |
139 | if (FindIntraL0Compaction( | |
140 | level_files, | |
141 | mutable_cf_options | |
142 | .level0_file_num_compaction_trigger /* min_files_to_compact */ | |
143 | , | |
f67539c2 TL |
144 | max_compact_bytes_per_del_file, |
145 | mutable_cf_options.max_compaction_bytes, &comp_inputs)) { | |
494da23a TL |
146 | Compaction* c = new Compaction( |
147 | vstorage, ioptions_, mutable_cf_options, {comp_inputs}, 0, | |
148 | 16 * 1024 * 1024 /* output file size limit */, | |
149 | 0 /* max compaction bytes, not applicable */, | |
150 | 0 /* output path ID */, mutable_cf_options.compression, | |
151 | ioptions_.compression_opts, 0 /* max_subcompactions */, {}, | |
152 | /* is manual */ false, vstorage->CompactionScore(0), | |
153 | /* is deletion compaction */ false, | |
154 | CompactionReason::kFIFOReduceNumFiles); | |
155 | return c; | |
156 | } | |
157 | } | |
158 | ||
159 | ROCKS_LOG_BUFFER( | |
160 | log_buffer, | |
161 | "[%s] FIFO compaction: nothing to do. Total size %" PRIu64 | |
162 | ", max size %" PRIu64 "\n", | |
163 | cf_name.c_str(), total_size, | |
164 | mutable_cf_options.compaction_options_fifo.max_table_files_size); | |
165 | return nullptr; | |
166 | } | |
167 | ||
168 | if (!level0_compactions_in_progress_.empty()) { | |
169 | ROCKS_LOG_BUFFER( | |
170 | log_buffer, | |
171 | "[%s] FIFO compaction: Already executing compaction. No need " | |
172 | "to run parallel compactions since compactions are very fast", | |
173 | cf_name.c_str()); | |
174 | return nullptr; | |
175 | } | |
176 | ||
177 | std::vector<CompactionInputFiles> inputs; | |
178 | inputs.emplace_back(); | |
179 | inputs[0].level = 0; | |
180 | ||
181 | for (auto ritr = level_files.rbegin(); ritr != level_files.rend(); ++ritr) { | |
182 | auto f = *ritr; | |
183 | total_size -= f->compensated_file_size; | |
184 | inputs[0].files.push_back(f); | |
185 | char tmp_fsize[16]; | |
186 | AppendHumanBytes(f->fd.GetFileSize(), tmp_fsize, sizeof(tmp_fsize)); | |
187 | ROCKS_LOG_BUFFER(log_buffer, | |
188 | "[%s] FIFO compaction: picking file %" PRIu64 | |
189 | " with size %s for deletion", | |
190 | cf_name.c_str(), f->fd.GetNumber(), tmp_fsize); | |
191 | if (total_size <= | |
192 | mutable_cf_options.compaction_options_fifo.max_table_files_size) { | |
193 | break; | |
194 | } | |
195 | } | |
196 | ||
197 | Compaction* c = new Compaction( | |
198 | vstorage, ioptions_, mutable_cf_options, std::move(inputs), 0, 0, 0, 0, | |
199 | kNoCompression, ioptions_.compression_opts, /* max_subcompactions */ 0, | |
200 | {}, /* is manual */ false, vstorage->CompactionScore(0), | |
201 | /* is deletion compaction */ true, CompactionReason::kFIFOMaxSize); | |
202 | return c; | |
203 | } | |
204 | ||
205 | Compaction* FIFOCompactionPicker::PickCompaction( | |
206 | const std::string& cf_name, const MutableCFOptions& mutable_cf_options, | |
f67539c2 TL |
207 | VersionStorageInfo* vstorage, LogBuffer* log_buffer, |
208 | SequenceNumber /*earliest_memtable_seqno*/) { | |
494da23a TL |
209 | assert(vstorage->num_levels() == 1); |
210 | ||
211 | Compaction* c = nullptr; | |
212 | if (mutable_cf_options.ttl > 0) { | |
213 | c = PickTTLCompaction(cf_name, mutable_cf_options, vstorage, log_buffer); | |
214 | } | |
215 | if (c == nullptr) { | |
216 | c = PickSizeCompaction(cf_name, mutable_cf_options, vstorage, log_buffer); | |
217 | } | |
218 | RegisterCompaction(c); | |
219 | return c; | |
220 | } | |
221 | ||
222 | Compaction* FIFOCompactionPicker::CompactRange( | |
223 | const std::string& cf_name, const MutableCFOptions& mutable_cf_options, | |
224 | VersionStorageInfo* vstorage, int input_level, int output_level, | |
f67539c2 | 225 | const CompactRangeOptions& /*compact_range_options*/, |
494da23a | 226 | const InternalKey* /*begin*/, const InternalKey* /*end*/, |
f67539c2 TL |
227 | InternalKey** compaction_end, bool* /*manual_conflict*/, |
228 | uint64_t /*max_file_num_to_ignore*/) { | |
494da23a TL |
229 | #ifdef NDEBUG |
230 | (void)input_level; | |
231 | (void)output_level; | |
232 | #endif | |
233 | assert(input_level == 0); | |
234 | assert(output_level == 0); | |
235 | *compaction_end = nullptr; | |
236 | LogBuffer log_buffer(InfoLogLevel::INFO_LEVEL, ioptions_.info_log); | |
237 | Compaction* c = | |
238 | PickCompaction(cf_name, mutable_cf_options, vstorage, &log_buffer); | |
239 | log_buffer.FlushBufferToLog(); | |
240 | return c; | |
241 | } | |
242 | ||
f67539c2 | 243 | } // namespace ROCKSDB_NAMESPACE |
494da23a | 244 | #endif // !ROCKSDB_LITE |