]>
Commit | Line | Data |
---|---|---|
7c673cae | 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. |
11fdf7f2 TL |
2 | // This source code is licensed under both the GPLv2 (found in the |
3 | // COPYING file in the root directory) and Apache 2.0 License | |
4 | // (found in the LICENSE.Apache file in the root directory). | |
7c673cae | 5 | // |
f67539c2 | 6 | #include "file/file_util.h" |
7c673cae FG |
7 | |
8 | #include <string> | |
9 | #include <algorithm> | |
10 | ||
f67539c2 TL |
11 | #include "file/random_access_file_reader.h" |
12 | #include "file/sequence_file_reader.h" | |
13 | #include "file/sst_file_manager_impl.h" | |
14 | #include "file/writable_file_writer.h" | |
7c673cae | 15 | #include "rocksdb/env.h" |
7c673cae | 16 | |
f67539c2 | 17 | namespace ROCKSDB_NAMESPACE { |
7c673cae FG |
18 | |
19 | // Utility function to copy a file up to a specified length | |
20effc67 TL |
20 | IOStatus CopyFile(FileSystem* fs, const std::string& source, |
21 | const std::string& destination, uint64_t size, bool use_fsync, | |
22 | const std::shared_ptr<IOTracer>& io_tracer) { | |
f67539c2 | 23 | const FileOptions soptions; |
20effc67 | 24 | IOStatus io_s; |
494da23a TL |
25 | std::unique_ptr<SequentialFileReader> src_reader; |
26 | std::unique_ptr<WritableFileWriter> dest_writer; | |
7c673cae FG |
27 | |
28 | { | |
f67539c2 | 29 | std::unique_ptr<FSSequentialFile> srcfile; |
20effc67 TL |
30 | io_s = fs->NewSequentialFile(source, soptions, &srcfile, nullptr); |
31 | if (!io_s.ok()) { | |
32 | return io_s; | |
11fdf7f2 | 33 | } |
f67539c2 | 34 | std::unique_ptr<FSWritableFile> destfile; |
20effc67 TL |
35 | io_s = fs->NewWritableFile(destination, soptions, &destfile, nullptr); |
36 | if (!io_s.ok()) { | |
37 | return io_s; | |
11fdf7f2 | 38 | } |
7c673cae | 39 | |
11fdf7f2 TL |
40 | if (size == 0) { |
41 | // default argument means copy everything | |
20effc67 TL |
42 | io_s = fs->GetFileSize(source, IOOptions(), &size, nullptr); |
43 | if (!io_s.ok()) { | |
44 | return io_s; | |
11fdf7f2 | 45 | } |
7c673cae | 46 | } |
20effc67 TL |
47 | src_reader.reset( |
48 | new SequentialFileReader(std::move(srcfile), source, io_tracer)); | |
11fdf7f2 TL |
49 | dest_writer.reset( |
50 | new WritableFileWriter(std::move(destfile), destination, soptions)); | |
7c673cae FG |
51 | } |
52 | ||
53 | char buffer[4096]; | |
54 | Slice slice; | |
55 | while (size > 0) { | |
56 | size_t bytes_to_read = std::min(sizeof(buffer), static_cast<size_t>(size)); | |
20effc67 TL |
57 | io_s = status_to_io_status(src_reader->Read(bytes_to_read, &slice, buffer)); |
58 | if (!io_s.ok()) { | |
59 | return io_s; | |
7c673cae | 60 | } |
11fdf7f2 | 61 | if (slice.size() == 0) { |
20effc67 | 62 | return IOStatus::Corruption("file too small"); |
7c673cae | 63 | } |
20effc67 TL |
64 | io_s = dest_writer->Append(slice); |
65 | if (!io_s.ok()) { | |
66 | return io_s; | |
7c673cae FG |
67 | } |
68 | size -= slice.size(); | |
69 | } | |
11fdf7f2 | 70 | return dest_writer->Sync(use_fsync); |
7c673cae FG |
71 | } |
72 | ||
73 | // Utility function to create a file with the provided contents | |
20effc67 TL |
74 | IOStatus CreateFile(FileSystem* fs, const std::string& destination, |
75 | const std::string& contents, bool use_fsync) { | |
7c673cae | 76 | const EnvOptions soptions; |
20effc67 | 77 | IOStatus io_s; |
494da23a | 78 | std::unique_ptr<WritableFileWriter> dest_writer; |
7c673cae | 79 | |
f67539c2 | 80 | std::unique_ptr<FSWritableFile> destfile; |
20effc67 TL |
81 | io_s = fs->NewWritableFile(destination, soptions, &destfile, nullptr); |
82 | if (!io_s.ok()) { | |
83 | return io_s; | |
7c673cae | 84 | } |
11fdf7f2 TL |
85 | dest_writer.reset( |
86 | new WritableFileWriter(std::move(destfile), destination, soptions)); | |
20effc67 TL |
87 | io_s = dest_writer->Append(Slice(contents)); |
88 | if (!io_s.ok()) { | |
89 | return io_s; | |
11fdf7f2 TL |
90 | } |
91 | return dest_writer->Sync(use_fsync); | |
7c673cae FG |
92 | } |
93 | ||
494da23a | 94 | Status DeleteDBFile(const ImmutableDBOptions* db_options, |
f67539c2 TL |
95 | const std::string& fname, const std::string& dir_to_sync, |
96 | const bool force_bg, const bool force_fg) { | |
7c673cae | 97 | #ifndef ROCKSDB_LITE |
494da23a | 98 | SstFileManagerImpl* sfm = |
7c673cae | 99 | static_cast<SstFileManagerImpl*>(db_options->sst_file_manager.get()); |
f67539c2 | 100 | if (sfm && !force_fg) { |
494da23a | 101 | return sfm->ScheduleFileDeletion(fname, dir_to_sync, force_bg); |
7c673cae FG |
102 | } else { |
103 | return db_options->env->DeleteFile(fname); | |
104 | } | |
105 | #else | |
11fdf7f2 | 106 | (void)dir_to_sync; |
494da23a | 107 | (void)force_bg; |
f67539c2 | 108 | (void)force_fg; |
7c673cae | 109 | // SstFileManager is not supported in ROCKSDB_LITE |
494da23a | 110 | // Delete file immediately |
7c673cae FG |
111 | return db_options->env->DeleteFile(fname); |
112 | #endif | |
113 | } | |
114 | ||
f67539c2 TL |
115 | bool IsWalDirSameAsDBPath(const ImmutableDBOptions* db_options) { |
116 | bool same = false; | |
117 | assert(!db_options->db_paths.empty()); | |
118 | Status s = db_options->env->AreFilesSame(db_options->wal_dir, | |
119 | db_options->db_paths[0].path, &same); | |
120 | if (s.IsNotSupported()) { | |
121 | same = db_options->wal_dir == db_options->db_paths[0].path; | |
122 | } | |
123 | return same; | |
124 | } | |
125 | ||
20effc67 TL |
126 | // requested_checksum_func_name brings the function name of the checksum |
127 | // generator in checksum_factory. Checksum factories may use or ignore | |
128 | // requested_checksum_func_name. | |
129 | IOStatus GenerateOneFileChecksum( | |
130 | FileSystem* fs, const std::string& file_path, | |
131 | FileChecksumGenFactory* checksum_factory, | |
132 | const std::string& requested_checksum_func_name, std::string* file_checksum, | |
133 | std::string* file_checksum_func_name, | |
134 | size_t verify_checksums_readahead_size, bool allow_mmap_reads, | |
135 | std::shared_ptr<IOTracer>& io_tracer) { | |
136 | if (checksum_factory == nullptr) { | |
137 | return IOStatus::InvalidArgument("Checksum factory is invalid"); | |
138 | } | |
139 | assert(file_checksum != nullptr); | |
140 | assert(file_checksum_func_name != nullptr); | |
141 | ||
142 | FileChecksumGenContext gen_context; | |
143 | gen_context.requested_checksum_func_name = requested_checksum_func_name; | |
144 | gen_context.file_name = file_path; | |
145 | std::unique_ptr<FileChecksumGenerator> checksum_generator = | |
146 | checksum_factory->CreateFileChecksumGenerator(gen_context); | |
147 | if (checksum_generator == nullptr) { | |
148 | std::string msg = | |
149 | "Cannot get the file checksum generator based on the requested " | |
150 | "checksum function name: " + | |
151 | requested_checksum_func_name + | |
152 | " from checksum factory: " + checksum_factory->Name(); | |
153 | return IOStatus::InvalidArgument(msg); | |
154 | } | |
155 | ||
156 | // For backward compatable, requested_checksum_func_name can be empty. | |
157 | // If we give the requested checksum function name, we expect it is the | |
158 | // same name of the checksum generator. | |
159 | assert(!checksum_generator || requested_checksum_func_name.empty() || | |
160 | requested_checksum_func_name == checksum_generator->Name()); | |
161 | ||
162 | uint64_t size; | |
163 | IOStatus io_s; | |
164 | std::unique_ptr<RandomAccessFileReader> reader; | |
165 | { | |
166 | std::unique_ptr<FSRandomAccessFile> r_file; | |
167 | io_s = fs->NewRandomAccessFile(file_path, FileOptions(), &r_file, nullptr); | |
168 | if (!io_s.ok()) { | |
169 | return io_s; | |
170 | } | |
171 | io_s = fs->GetFileSize(file_path, IOOptions(), &size, nullptr); | |
172 | if (!io_s.ok()) { | |
173 | return io_s; | |
174 | } | |
175 | reader.reset(new RandomAccessFileReader(std::move(r_file), file_path, | |
176 | nullptr /*Env*/, io_tracer)); | |
177 | } | |
178 | ||
179 | // Found that 256 KB readahead size provides the best performance, based on | |
180 | // experiments, for auto readahead. Experiment data is in PR #3282. | |
181 | size_t default_max_read_ahead_size = 256 * 1024; | |
182 | size_t readahead_size = (verify_checksums_readahead_size != 0) | |
183 | ? verify_checksums_readahead_size | |
184 | : default_max_read_ahead_size; | |
185 | ||
186 | FilePrefetchBuffer prefetch_buffer( | |
187 | reader.get(), readahead_size /* readadhead_size */, | |
188 | readahead_size /* max_readahead_size */, !allow_mmap_reads /* enable */); | |
189 | ||
190 | Slice slice; | |
191 | uint64_t offset = 0; | |
192 | IOOptions opts; | |
193 | while (size > 0) { | |
194 | size_t bytes_to_read = | |
195 | static_cast<size_t>(std::min(uint64_t{readahead_size}, size)); | |
196 | if (!prefetch_buffer.TryReadFromCache(opts, offset, bytes_to_read, &slice, | |
197 | false)) { | |
198 | return IOStatus::Corruption("file read failed"); | |
199 | } | |
200 | if (slice.size() == 0) { | |
201 | return IOStatus::Corruption("file too small"); | |
202 | } | |
203 | checksum_generator->Update(slice.data(), slice.size()); | |
204 | size -= slice.size(); | |
205 | offset += slice.size(); | |
206 | } | |
207 | checksum_generator->Finalize(); | |
208 | *file_checksum = checksum_generator->GetChecksum(); | |
209 | *file_checksum_func_name = checksum_generator->Name(); | |
210 | return IOStatus::OK(); | |
211 | } | |
212 | ||
213 | Status DestroyDir(Env* env, const std::string& dir) { | |
214 | Status s; | |
215 | if (env->FileExists(dir).IsNotFound()) { | |
216 | return s; | |
217 | } | |
218 | std::vector<std::string> files_in_dir; | |
219 | s = env->GetChildren(dir, &files_in_dir); | |
220 | if (s.ok()) { | |
221 | for (auto& file_in_dir : files_in_dir) { | |
222 | if (file_in_dir == "." || file_in_dir == "..") { | |
223 | continue; | |
224 | } | |
225 | std::string path = dir + "/" + file_in_dir; | |
226 | bool is_dir = false; | |
227 | s = env->IsDirectory(path, &is_dir); | |
228 | if (s.ok()) { | |
229 | if (is_dir) { | |
230 | s = DestroyDir(env, path); | |
231 | } else { | |
232 | s = env->DeleteFile(path); | |
233 | } | |
234 | } else if (s.IsNotSupported()) { | |
235 | s = Status::OK(); | |
236 | } | |
237 | if (!s.ok()) { | |
238 | // IsDirectory, etc. might not report NotFound | |
239 | if (s.IsNotFound() || env->FileExists(path).IsNotFound()) { | |
240 | // Allow files to be deleted externally | |
241 | s = Status::OK(); | |
242 | } else { | |
243 | break; | |
244 | } | |
245 | } | |
246 | } | |
247 | } | |
248 | ||
249 | if (s.ok()) { | |
250 | s = env->DeleteDir(dir); | |
251 | // DeleteDir might or might not report NotFound | |
252 | if (!s.ok() && (s.IsNotFound() || env->FileExists(dir).IsNotFound())) { | |
253 | // Allow to be deleted externally | |
254 | s = Status::OK(); | |
255 | } | |
256 | } | |
257 | return s; | |
258 | } | |
259 | ||
f67539c2 | 260 | } // namespace ROCKSDB_NAMESPACE |