1 // Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
2 // This source code is licensed under both the GPLv2 (found in the
3 // COPYING file in the root directory) and Apache 2.0 License
4 // (found in the LICENSE.Apache file in the root directory).
6 #include "file/file_util.h"
11 #include "file/random_access_file_reader.h"
12 #include "file/sequence_file_reader.h"
13 #include "file/sst_file_manager_impl.h"
14 #include "file/writable_file_writer.h"
15 #include "rocksdb/env.h"
17 namespace ROCKSDB_NAMESPACE
{
19 // Utility function to copy a file up to a specified length
20 IOStatus
CopyFile(FileSystem
* fs
, const std::string
& source
,
21 std::unique_ptr
<WritableFileWriter
>& dest_writer
,
22 uint64_t size
, bool use_fsync
,
23 const std::shared_ptr
<IOTracer
>& io_tracer
,
24 const Temperature temperature
) {
27 std::unique_ptr
<SequentialFileReader
> src_reader
;
30 soptions
.temperature
= temperature
;
31 std::unique_ptr
<FSSequentialFile
> srcfile
;
32 io_s
= fs
->NewSequentialFile(source
, soptions
, &srcfile
, nullptr);
38 // default argument means copy everything
39 io_s
= fs
->GetFileSize(source
, IOOptions(), &size
, nullptr);
45 new SequentialFileReader(std::move(srcfile
), source
, io_tracer
));
51 size_t bytes_to_read
= std::min(sizeof(buffer
), static_cast<size_t>(size
));
52 // TODO: rate limit copy file
53 io_s
= status_to_io_status(
54 src_reader
->Read(bytes_to_read
, &slice
, buffer
,
55 Env::IO_TOTAL
/* rate_limiter_priority */));
59 if (slice
.size() == 0) {
60 return IOStatus::Corruption("file too small");
62 io_s
= dest_writer
->Append(slice
);
68 return dest_writer
->Sync(use_fsync
);
71 IOStatus
CopyFile(FileSystem
* fs
, const std::string
& source
,
72 const std::string
& destination
, uint64_t size
, bool use_fsync
,
73 const std::shared_ptr
<IOTracer
>& io_tracer
,
74 const Temperature temperature
) {
77 std::unique_ptr
<WritableFileWriter
> dest_writer
;
80 options
.temperature
= temperature
;
81 std::unique_ptr
<FSWritableFile
> destfile
;
82 io_s
= fs
->NewWritableFile(destination
, options
, &destfile
, nullptr);
88 new WritableFileWriter(std::move(destfile
), destination
, options
));
91 return CopyFile(fs
, source
, dest_writer
, size
, use_fsync
, io_tracer
,
95 // Utility function to create a file with the provided contents
96 IOStatus
CreateFile(FileSystem
* fs
, const std::string
& destination
,
97 const std::string
& contents
, bool use_fsync
) {
98 const EnvOptions soptions
;
100 std::unique_ptr
<WritableFileWriter
> dest_writer
;
102 std::unique_ptr
<FSWritableFile
> destfile
;
103 io_s
= fs
->NewWritableFile(destination
, soptions
, &destfile
, nullptr);
108 new WritableFileWriter(std::move(destfile
), destination
, soptions
));
109 io_s
= dest_writer
->Append(Slice(contents
));
113 return dest_writer
->Sync(use_fsync
);
116 Status
DeleteDBFile(const ImmutableDBOptions
* db_options
,
117 const std::string
& fname
, const std::string
& dir_to_sync
,
118 const bool force_bg
, const bool force_fg
) {
120 SstFileManagerImpl
* sfm
=
121 static_cast<SstFileManagerImpl
*>(db_options
->sst_file_manager
.get());
122 if (sfm
&& !force_fg
) {
123 return sfm
->ScheduleFileDeletion(fname
, dir_to_sync
, force_bg
);
125 return db_options
->env
->DeleteFile(fname
);
131 // SstFileManager is not supported in ROCKSDB_LITE
132 // Delete file immediately
133 return db_options
->env
->DeleteFile(fname
);
137 // requested_checksum_func_name brings the function name of the checksum
138 // generator in checksum_factory. Empty string is permitted, in which case the
139 // name of the generator created by the factory is unchecked. When
140 // `requested_checksum_func_name` is non-empty, however, the created generator's
141 // name must match it, otherwise an `InvalidArgument` error is returned.
142 IOStatus
GenerateOneFileChecksum(
143 FileSystem
* fs
, const std::string
& file_path
,
144 FileChecksumGenFactory
* checksum_factory
,
145 const std::string
& requested_checksum_func_name
, std::string
* file_checksum
,
146 std::string
* file_checksum_func_name
,
147 size_t verify_checksums_readahead_size
, bool allow_mmap_reads
,
148 std::shared_ptr
<IOTracer
>& io_tracer
, RateLimiter
* rate_limiter
,
149 Env::IOPriority rate_limiter_priority
) {
150 if (checksum_factory
== nullptr) {
151 return IOStatus::InvalidArgument("Checksum factory is invalid");
153 assert(file_checksum
!= nullptr);
154 assert(file_checksum_func_name
!= nullptr);
156 FileChecksumGenContext gen_context
;
157 gen_context
.requested_checksum_func_name
= requested_checksum_func_name
;
158 gen_context
.file_name
= file_path
;
159 std::unique_ptr
<FileChecksumGenerator
> checksum_generator
=
160 checksum_factory
->CreateFileChecksumGenerator(gen_context
);
161 if (checksum_generator
== nullptr) {
163 "Cannot get the file checksum generator based on the requested "
164 "checksum function name: " +
165 requested_checksum_func_name
+
166 " from checksum factory: " + checksum_factory
->Name();
167 return IOStatus::InvalidArgument(msg
);
169 // For backward compatibility and use in file ingestion clients where there
170 // is no stored checksum function name, `requested_checksum_func_name` can
171 // be empty. If we give the requested checksum function name, we expect it
172 // is the same name of the checksum generator.
173 if (!requested_checksum_func_name
.empty() &&
174 checksum_generator
->Name() != requested_checksum_func_name
) {
175 std::string msg
= "Expected file checksum generator named '" +
176 requested_checksum_func_name
+
177 "', while the factory created one "
179 checksum_generator
->Name() + "'";
180 return IOStatus::InvalidArgument(msg
);
186 std::unique_ptr
<RandomAccessFileReader
> reader
;
188 std::unique_ptr
<FSRandomAccessFile
> r_file
;
189 io_s
= fs
->NewRandomAccessFile(file_path
, FileOptions(), &r_file
, nullptr);
193 io_s
= fs
->GetFileSize(file_path
, IOOptions(), &size
, nullptr);
197 reader
.reset(new RandomAccessFileReader(std::move(r_file
), file_path
,
198 nullptr /*Env*/, io_tracer
, nullptr,
199 0, nullptr, rate_limiter
));
202 // Found that 256 KB readahead size provides the best performance, based on
203 // experiments, for auto readahead. Experiment data is in PR #3282.
204 size_t default_max_read_ahead_size
= 256 * 1024;
205 size_t readahead_size
= (verify_checksums_readahead_size
!= 0)
206 ? verify_checksums_readahead_size
207 : default_max_read_ahead_size
;
209 FilePrefetchBuffer
prefetch_buffer(readahead_size
/* readahead_size */,
210 readahead_size
/* max_readahead_size */,
211 !allow_mmap_reads
/* enable */);
217 size_t bytes_to_read
=
218 static_cast<size_t>(std::min(uint64_t{readahead_size
}, size
));
219 if (!prefetch_buffer
.TryReadFromCache(
220 opts
, reader
.get(), offset
, bytes_to_read
, &slice
,
221 nullptr /* status */, rate_limiter_priority
,
222 false /* for_compaction */)) {
223 return IOStatus::Corruption("file read failed");
225 if (slice
.size() == 0) {
226 return IOStatus::Corruption("file too small");
228 checksum_generator
->Update(slice
.data(), slice
.size());
229 size
-= slice
.size();
230 offset
+= slice
.size();
232 checksum_generator
->Finalize();
233 *file_checksum
= checksum_generator
->GetChecksum();
234 *file_checksum_func_name
= checksum_generator
->Name();
235 return IOStatus::OK();
238 Status
DestroyDir(Env
* env
, const std::string
& dir
) {
240 if (env
->FileExists(dir
).IsNotFound()) {
243 std::vector
<std::string
> files_in_dir
;
244 s
= env
->GetChildren(dir
, &files_in_dir
);
246 for (auto& file_in_dir
: files_in_dir
) {
247 std::string path
= dir
+ "/" + file_in_dir
;
249 s
= env
->IsDirectory(path
, &is_dir
);
252 s
= DestroyDir(env
, path
);
254 s
= env
->DeleteFile(path
);
256 } else if (s
.IsNotSupported()) {
260 // IsDirectory, etc. might not report NotFound
261 if (s
.IsNotFound() || env
->FileExists(path
).IsNotFound()) {
262 // Allow files to be deleted externally
272 s
= env
->DeleteDir(dir
);
273 // DeleteDir might or might not report NotFound
274 if (!s
.ok() && (s
.IsNotFound() || env
->FileExists(dir
).IsNotFound())) {
275 // Allow to be deleted externally
282 } // namespace ROCKSDB_NAMESPACE