]>
Commit | Line | Data |
---|---|---|
7c673cae | 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. |
11fdf7f2 TL |
2 | // This source code is licensed under both the GPLv2 (found in the |
3 | // COPYING file in the root directory) and Apache 2.0 License | |
4 | // (found in the LICENSE.Apache file in the root directory). | |
7c673cae | 5 | // |
f67539c2 | 6 | #include "file/file_util.h" |
7c673cae | 7 | |
7c673cae | 8 | #include <algorithm> |
1e59de90 | 9 | #include <string> |
7c673cae | 10 | |
f67539c2 TL |
11 | #include "file/random_access_file_reader.h" |
12 | #include "file/sequence_file_reader.h" | |
13 | #include "file/sst_file_manager_impl.h" | |
14 | #include "file/writable_file_writer.h" | |
7c673cae | 15 | #include "rocksdb/env.h" |
7c673cae | 16 | |
f67539c2 | 17 | namespace ROCKSDB_NAMESPACE { |
7c673cae FG |
18 | |
19 | // Utility function to copy a file up to a specified length | |
20effc67 | 20 | IOStatus CopyFile(FileSystem* fs, const std::string& source, |
1e59de90 TL |
21 | std::unique_ptr<WritableFileWriter>& dest_writer, |
22 | uint64_t size, bool use_fsync, | |
23 | const std::shared_ptr<IOTracer>& io_tracer, | |
24 | const Temperature temperature) { | |
25 | FileOptions soptions; | |
20effc67 | 26 | IOStatus io_s; |
494da23a | 27 | std::unique_ptr<SequentialFileReader> src_reader; |
7c673cae FG |
28 | |
29 | { | |
1e59de90 | 30 | soptions.temperature = temperature; |
f67539c2 | 31 | std::unique_ptr<FSSequentialFile> srcfile; |
20effc67 TL |
32 | io_s = fs->NewSequentialFile(source, soptions, &srcfile, nullptr); |
33 | if (!io_s.ok()) { | |
34 | return io_s; | |
11fdf7f2 | 35 | } |
7c673cae | 36 | |
11fdf7f2 TL |
37 | if (size == 0) { |
38 | // default argument means copy everything | |
20effc67 TL |
39 | io_s = fs->GetFileSize(source, IOOptions(), &size, nullptr); |
40 | if (!io_s.ok()) { | |
41 | return io_s; | |
11fdf7f2 | 42 | } |
7c673cae | 43 | } |
20effc67 TL |
44 | src_reader.reset( |
45 | new SequentialFileReader(std::move(srcfile), source, io_tracer)); | |
7c673cae FG |
46 | } |
47 | ||
48 | char buffer[4096]; | |
49 | Slice slice; | |
50 | while (size > 0) { | |
51 | size_t bytes_to_read = std::min(sizeof(buffer), static_cast<size_t>(size)); | |
1e59de90 TL |
52 | // TODO: rate limit copy file |
53 | io_s = status_to_io_status( | |
54 | src_reader->Read(bytes_to_read, &slice, buffer, | |
55 | Env::IO_TOTAL /* rate_limiter_priority */)); | |
20effc67 TL |
56 | if (!io_s.ok()) { |
57 | return io_s; | |
7c673cae | 58 | } |
11fdf7f2 | 59 | if (slice.size() == 0) { |
20effc67 | 60 | return IOStatus::Corruption("file too small"); |
7c673cae | 61 | } |
20effc67 TL |
62 | io_s = dest_writer->Append(slice); |
63 | if (!io_s.ok()) { | |
64 | return io_s; | |
7c673cae FG |
65 | } |
66 | size -= slice.size(); | |
67 | } | |
11fdf7f2 | 68 | return dest_writer->Sync(use_fsync); |
7c673cae FG |
69 | } |
70 | ||
1e59de90 TL |
71 | IOStatus CopyFile(FileSystem* fs, const std::string& source, |
72 | const std::string& destination, uint64_t size, bool use_fsync, | |
73 | const std::shared_ptr<IOTracer>& io_tracer, | |
74 | const Temperature temperature) { | |
75 | FileOptions options; | |
76 | IOStatus io_s; | |
77 | std::unique_ptr<WritableFileWriter> dest_writer; | |
78 | ||
79 | { | |
80 | options.temperature = temperature; | |
81 | std::unique_ptr<FSWritableFile> destfile; | |
82 | io_s = fs->NewWritableFile(destination, options, &destfile, nullptr); | |
83 | if (!io_s.ok()) { | |
84 | return io_s; | |
85 | } | |
86 | ||
87 | dest_writer.reset( | |
88 | new WritableFileWriter(std::move(destfile), destination, options)); | |
89 | } | |
90 | ||
91 | return CopyFile(fs, source, dest_writer, size, use_fsync, io_tracer, | |
92 | temperature); | |
93 | } | |
94 | ||
7c673cae | 95 | // Utility function to create a file with the provided contents |
20effc67 TL |
96 | IOStatus CreateFile(FileSystem* fs, const std::string& destination, |
97 | const std::string& contents, bool use_fsync) { | |
7c673cae | 98 | const EnvOptions soptions; |
20effc67 | 99 | IOStatus io_s; |
494da23a | 100 | std::unique_ptr<WritableFileWriter> dest_writer; |
7c673cae | 101 | |
f67539c2 | 102 | std::unique_ptr<FSWritableFile> destfile; |
20effc67 TL |
103 | io_s = fs->NewWritableFile(destination, soptions, &destfile, nullptr); |
104 | if (!io_s.ok()) { | |
105 | return io_s; | |
7c673cae | 106 | } |
11fdf7f2 TL |
107 | dest_writer.reset( |
108 | new WritableFileWriter(std::move(destfile), destination, soptions)); | |
20effc67 TL |
109 | io_s = dest_writer->Append(Slice(contents)); |
110 | if (!io_s.ok()) { | |
111 | return io_s; | |
11fdf7f2 TL |
112 | } |
113 | return dest_writer->Sync(use_fsync); | |
7c673cae FG |
114 | } |
115 | ||
494da23a | 116 | Status DeleteDBFile(const ImmutableDBOptions* db_options, |
f67539c2 TL |
117 | const std::string& fname, const std::string& dir_to_sync, |
118 | const bool force_bg, const bool force_fg) { | |
7c673cae | 119 | #ifndef ROCKSDB_LITE |
494da23a | 120 | SstFileManagerImpl* sfm = |
7c673cae | 121 | static_cast<SstFileManagerImpl*>(db_options->sst_file_manager.get()); |
f67539c2 | 122 | if (sfm && !force_fg) { |
494da23a | 123 | return sfm->ScheduleFileDeletion(fname, dir_to_sync, force_bg); |
7c673cae FG |
124 | } else { |
125 | return db_options->env->DeleteFile(fname); | |
126 | } | |
127 | #else | |
11fdf7f2 | 128 | (void)dir_to_sync; |
494da23a | 129 | (void)force_bg; |
f67539c2 | 130 | (void)force_fg; |
7c673cae | 131 | // SstFileManager is not supported in ROCKSDB_LITE |
494da23a | 132 | // Delete file immediately |
7c673cae FG |
133 | return db_options->env->DeleteFile(fname); |
134 | #endif | |
135 | } | |
136 | ||
20effc67 | 137 | // requested_checksum_func_name brings the function name of the checksum |
1e59de90 TL |
138 | // generator in checksum_factory. Empty string is permitted, in which case the |
139 | // name of the generator created by the factory is unchecked. When | |
140 | // `requested_checksum_func_name` is non-empty, however, the created generator's | |
141 | // name must match it, otherwise an `InvalidArgument` error is returned. | |
20effc67 TL |
142 | IOStatus GenerateOneFileChecksum( |
143 | FileSystem* fs, const std::string& file_path, | |
144 | FileChecksumGenFactory* checksum_factory, | |
145 | const std::string& requested_checksum_func_name, std::string* file_checksum, | |
146 | std::string* file_checksum_func_name, | |
147 | size_t verify_checksums_readahead_size, bool allow_mmap_reads, | |
1e59de90 TL |
148 | std::shared_ptr<IOTracer>& io_tracer, RateLimiter* rate_limiter, |
149 | Env::IOPriority rate_limiter_priority) { | |
20effc67 TL |
150 | if (checksum_factory == nullptr) { |
151 | return IOStatus::InvalidArgument("Checksum factory is invalid"); | |
152 | } | |
153 | assert(file_checksum != nullptr); | |
154 | assert(file_checksum_func_name != nullptr); | |
155 | ||
156 | FileChecksumGenContext gen_context; | |
157 | gen_context.requested_checksum_func_name = requested_checksum_func_name; | |
158 | gen_context.file_name = file_path; | |
159 | std::unique_ptr<FileChecksumGenerator> checksum_generator = | |
160 | checksum_factory->CreateFileChecksumGenerator(gen_context); | |
161 | if (checksum_generator == nullptr) { | |
162 | std::string msg = | |
163 | "Cannot get the file checksum generator based on the requested " | |
164 | "checksum function name: " + | |
165 | requested_checksum_func_name + | |
166 | " from checksum factory: " + checksum_factory->Name(); | |
167 | return IOStatus::InvalidArgument(msg); | |
1e59de90 TL |
168 | } else { |
169 | // For backward compatibility and use in file ingestion clients where there | |
170 | // is no stored checksum function name, `requested_checksum_func_name` can | |
171 | // be empty. If we give the requested checksum function name, we expect it | |
172 | // is the same name of the checksum generator. | |
173 | if (!requested_checksum_func_name.empty() && | |
174 | checksum_generator->Name() != requested_checksum_func_name) { | |
175 | std::string msg = "Expected file checksum generator named '" + | |
176 | requested_checksum_func_name + | |
177 | "', while the factory created one " | |
178 | "named '" + | |
179 | checksum_generator->Name() + "'"; | |
180 | return IOStatus::InvalidArgument(msg); | |
181 | } | |
20effc67 TL |
182 | } |
183 | ||
20effc67 TL |
184 | uint64_t size; |
185 | IOStatus io_s; | |
186 | std::unique_ptr<RandomAccessFileReader> reader; | |
187 | { | |
188 | std::unique_ptr<FSRandomAccessFile> r_file; | |
189 | io_s = fs->NewRandomAccessFile(file_path, FileOptions(), &r_file, nullptr); | |
190 | if (!io_s.ok()) { | |
191 | return io_s; | |
192 | } | |
193 | io_s = fs->GetFileSize(file_path, IOOptions(), &size, nullptr); | |
194 | if (!io_s.ok()) { | |
195 | return io_s; | |
196 | } | |
197 | reader.reset(new RandomAccessFileReader(std::move(r_file), file_path, | |
1e59de90 TL |
198 | nullptr /*Env*/, io_tracer, nullptr, |
199 | 0, nullptr, rate_limiter)); | |
20effc67 TL |
200 | } |
201 | ||
202 | // Found that 256 KB readahead size provides the best performance, based on | |
203 | // experiments, for auto readahead. Experiment data is in PR #3282. | |
204 | size_t default_max_read_ahead_size = 256 * 1024; | |
205 | size_t readahead_size = (verify_checksums_readahead_size != 0) | |
206 | ? verify_checksums_readahead_size | |
207 | : default_max_read_ahead_size; | |
208 | ||
1e59de90 TL |
209 | FilePrefetchBuffer prefetch_buffer(readahead_size /* readahead_size */, |
210 | readahead_size /* max_readahead_size */, | |
211 | !allow_mmap_reads /* enable */); | |
20effc67 TL |
212 | |
213 | Slice slice; | |
214 | uint64_t offset = 0; | |
215 | IOOptions opts; | |
216 | while (size > 0) { | |
217 | size_t bytes_to_read = | |
218 | static_cast<size_t>(std::min(uint64_t{readahead_size}, size)); | |
1e59de90 TL |
219 | if (!prefetch_buffer.TryReadFromCache( |
220 | opts, reader.get(), offset, bytes_to_read, &slice, | |
221 | nullptr /* status */, rate_limiter_priority, | |
222 | false /* for_compaction */)) { | |
20effc67 TL |
223 | return IOStatus::Corruption("file read failed"); |
224 | } | |
225 | if (slice.size() == 0) { | |
226 | return IOStatus::Corruption("file too small"); | |
227 | } | |
228 | checksum_generator->Update(slice.data(), slice.size()); | |
229 | size -= slice.size(); | |
230 | offset += slice.size(); | |
231 | } | |
232 | checksum_generator->Finalize(); | |
233 | *file_checksum = checksum_generator->GetChecksum(); | |
234 | *file_checksum_func_name = checksum_generator->Name(); | |
235 | return IOStatus::OK(); | |
236 | } | |
237 | ||
238 | Status DestroyDir(Env* env, const std::string& dir) { | |
239 | Status s; | |
240 | if (env->FileExists(dir).IsNotFound()) { | |
241 | return s; | |
242 | } | |
243 | std::vector<std::string> files_in_dir; | |
244 | s = env->GetChildren(dir, &files_in_dir); | |
245 | if (s.ok()) { | |
246 | for (auto& file_in_dir : files_in_dir) { | |
20effc67 TL |
247 | std::string path = dir + "/" + file_in_dir; |
248 | bool is_dir = false; | |
249 | s = env->IsDirectory(path, &is_dir); | |
250 | if (s.ok()) { | |
251 | if (is_dir) { | |
252 | s = DestroyDir(env, path); | |
253 | } else { | |
254 | s = env->DeleteFile(path); | |
255 | } | |
256 | } else if (s.IsNotSupported()) { | |
257 | s = Status::OK(); | |
258 | } | |
259 | if (!s.ok()) { | |
260 | // IsDirectory, etc. might not report NotFound | |
261 | if (s.IsNotFound() || env->FileExists(path).IsNotFound()) { | |
262 | // Allow files to be deleted externally | |
263 | s = Status::OK(); | |
264 | } else { | |
265 | break; | |
266 | } | |
267 | } | |
268 | } | |
269 | } | |
270 | ||
271 | if (s.ok()) { | |
272 | s = env->DeleteDir(dir); | |
273 | // DeleteDir might or might not report NotFound | |
274 | if (!s.ok() && (s.IsNotFound() || env->FileExists(dir).IsNotFound())) { | |
275 | // Allow to be deleted externally | |
276 | s = Status::OK(); | |
277 | } | |
278 | } | |
279 | return s; | |
280 | } | |
281 | ||
f67539c2 | 282 | } // namespace ROCKSDB_NAMESPACE |