1 // Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
2 // This source code is licensed under both the GPLv2 (found in the
3 // COPYING file in the root directory) and Apache 2.0 License
4 // (found in the LICENSE.Apache file in the root directory).
5 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
6 // Use of this source code is governed by a BSD-style license that can be
7 // found in the LICENSE file. See the AUTHORS file for names of contributors.
9 // An Env is an interface used by the rocksdb implementation to access
10 // operating system functionality like the filesystem etc. Callers
11 // may wish to provide a custom Env object when opening a database to
12 // get fine gain control; e.g., to rate limit file system operations.
14 // All Env implementations are safe for concurrent access from
15 // multiple threads without any external synchronization.
28 #include "rocksdb/customizable.h"
29 #include "rocksdb/functor_wrapper.h"
30 #include "rocksdb/status.h"
31 #include "rocksdb/thread_status.h"
34 // Windows API macro interference
40 #if defined(__GNUC__) || defined(__clang__)
41 #define ROCKSDB_PRINTF_FORMAT_ATTR(format_param, dots_param) \
42 __attribute__((__format__(__printf__, format_param, dots_param)))
44 #define ROCKSDB_PRINTF_FORMAT_ATTR(format_param, dots_param)
47 namespace ROCKSDB_NAMESPACE
{
52 class RandomAccessFile
;
55 struct DataVerificationInfo
;
58 class MemoryMappedFileBuffer
;
61 struct ImmutableDBOptions
;
62 struct MutableDBOptions
;
64 class ThreadStatusUpdater
;
70 const size_t kDefaultPageSize
= 4 * 1024;
72 enum class CpuPriority
{
79 // Options while opening a file to read/write
81 // Construct with default Options
84 // Construct from Options
85 explicit EnvOptions(const DBOptions
& options
);
87 // If true, then use mmap to read data.
88 // Not recommended for 32-bit OS.
89 bool use_mmap_reads
= false;
91 // If true, then use mmap to write data
92 bool use_mmap_writes
= true;
94 // If true, then use O_DIRECT for reading data
95 bool use_direct_reads
= false;
97 // If true, then use O_DIRECT for writing data
98 bool use_direct_writes
= false;
100 // If false, fallocate() calls are bypassed
101 bool allow_fallocate
= true;
103 // If true, set the FD_CLOEXEC on open fd.
104 bool set_fd_cloexec
= true;
106 // Allows OS to incrementally sync files to disk while they are being
107 // written, in the background. Issue one request for every bytes_per_sync
108 // written. 0 turns it off.
110 uint64_t bytes_per_sync
= 0;
112 // When true, guarantees the file has at most `bytes_per_sync` bytes submitted
113 // for writeback at any given time.
115 // - If `sync_file_range` is supported it achieves this by waiting for any
116 // prior `sync_file_range`s to finish before proceeding. In this way,
117 // processing (compression, etc.) can proceed uninhibited in the gap
118 // between `sync_file_range`s, and we block only when I/O falls behind.
119 // - Otherwise the `WritableFile::Sync` method is used. Note this mechanism
120 // always blocks, thus preventing the interleaving of I/O and processing.
122 // Note: Enabling this option does not provide any additional persistence
123 // guarantees, as it may use `sync_file_range`, which does not write out
127 bool strict_bytes_per_sync
= false;
129 // If true, we will preallocate the file with FALLOC_FL_KEEP_SIZE flag, which
130 // means that file size won't change as part of preallocation.
131 // If false, preallocation will also change the file size. This option will
132 // improve the performance in workloads where you sync the data on every
133 // write. By default, we set it to true for MANIFEST writes and false for
135 bool fallocate_with_keep_size
= true;
138 size_t compaction_readahead_size
= 0;
141 size_t random_access_max_buffer_size
= 0;
144 size_t writable_file_max_buffer_size
= 1024 * 1024;
146 // If not nullptr, write rate limiting is enabled for flush and compaction
147 RateLimiter
* rate_limiter
= nullptr;
150 // Exceptions MUST NOT propagate out of overridden functions into RocksDB,
151 // because RocksDB is not exception-safe. This could cause undefined behavior
152 // including data loss, unreported corruption, deadlocks, and more.
153 class Env
: public Customizable
{
155 static const char* kDefaultName() { return "DefaultEnv"; }
156 struct FileAttributes
{
160 // Size of file in bytes
165 // Construct an Env with a separate FileSystem and/or SystemClock
167 explicit Env(const std::shared_ptr
<FileSystem
>& fs
);
168 Env(const std::shared_ptr
<FileSystem
>& fs
,
169 const std::shared_ptr
<SystemClock
>& clock
);
170 // No copying allowed
171 Env(const Env
&) = delete;
172 void operator=(const Env
&) = delete;
176 static const char* Type() { return "Environment"; }
178 // Deprecated. Will be removed in a major release. Derived classes
179 // should implement this method.
180 const char* Name() const override
{ return ""; }
182 // Loads the environment specified by the input value into the result
183 // The CreateFromString alternative should be used; this method may be
184 // deprecated in a future release.
185 static Status
LoadEnv(const std::string
& value
, Env
** result
);
187 // Loads the environment specified by the input value into the result
188 // The CreateFromString alternative should be used; this method may be
189 // deprecated in a future release.
190 static Status
LoadEnv(const std::string
& value
, Env
** result
,
191 std::shared_ptr
<Env
>* guard
);
193 // Loads the environment specified by the input value into the result
194 // @see Customizable for a more detailed description of the parameters and
197 // @param config_options Controls how the environment is loaded.
198 // @param value the name and associated properties for the environment.
199 // @param result On success, the environment that was loaded.
200 // @param guard If specified and the loaded environment is not static,
201 // this value will contain the loaded environment (guard.get() ==
203 // @return OK If the environment was successfully loaded (and optionally
205 // @return not-OK if the load failed.
206 static Status
CreateFromString(const ConfigOptions
& config_options
,
207 const std::string
& value
, Env
** result
);
208 static Status
CreateFromString(const ConfigOptions
& config_options
,
209 const std::string
& value
, Env
** result
,
210 std::shared_ptr
<Env
>* guard
);
212 // Loads the environment specified by the env and fs uri.
213 // If both are specified, an error is returned.
214 // Otherwise, the environment is created by loading (via CreateFromString)
215 // the appropriate env/fs from the corresponding values.
216 static Status
CreateFromUri(const ConfigOptions
& options
,
217 const std::string
& env_uri
,
218 const std::string
& fs_uri
, Env
** result
,
219 std::shared_ptr
<Env
>* guard
);
221 // Return a default environment suitable for the current operating
222 // system. Sophisticated users may wish to provide their own Env
223 // implementation instead of relying on this default environment.
225 // The result of Default() belongs to rocksdb and must never be deleted.
226 static Env
* Default();
228 // See FileSystem::RegisterDbPaths.
229 virtual Status
RegisterDbPaths(const std::vector
<std::string
>& /*paths*/) {
232 // See FileSystem::UnregisterDbPaths.
233 virtual Status
UnregisterDbPaths(const std::vector
<std::string
>& /*paths*/) {
237 // Create a brand new sequentially-readable file with the specified name.
238 // On success, stores a pointer to the new file in *result and returns OK.
239 // On failure stores nullptr in *result and returns non-OK. If the file does
240 // not exist, returns a non-OK status.
242 // The returned file will only be accessed by one thread at a time.
243 virtual Status
NewSequentialFile(const std::string
& fname
,
244 std::unique_ptr
<SequentialFile
>* result
,
245 const EnvOptions
& options
) = 0;
247 // Create a brand new random access read-only file with the
248 // specified name. On success, stores a pointer to the new file in
249 // *result and returns OK. On failure stores nullptr in *result and
250 // returns non-OK. If the file does not exist, returns a non-OK
253 // The returned file may be concurrently accessed by multiple threads.
254 virtual Status
NewRandomAccessFile(const std::string
& fname
,
255 std::unique_ptr
<RandomAccessFile
>* result
,
256 const EnvOptions
& options
) = 0;
257 // These values match Linux definition
258 // https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/include/uapi/linux/fcntl.h#n56
259 enum WriteLifeTimeHint
{
260 WLTH_NOT_SET
= 0, // No hint information set
261 WLTH_NONE
, // No hints about write life time
262 WLTH_SHORT
, // Data written has a short life time
263 WLTH_MEDIUM
, // Data written has a medium life time
264 WLTH_LONG
, // Data written has a long life time
265 WLTH_EXTREME
, // Data written has an extremely long life time
268 // Create an object that writes to a new file with the specified
269 // name. Deletes any existing file with the same name and creates a
270 // new file. On success, stores a pointer to the new file in
271 // *result and returns OK. On failure stores nullptr in *result and
274 // The returned file will only be accessed by one thread at a time.
275 virtual Status
NewWritableFile(const std::string
& fname
,
276 std::unique_ptr
<WritableFile
>* result
,
277 const EnvOptions
& options
) = 0;
279 // Create an object that writes to a file with the specified name.
280 // `WritableFile::Append()`s will append after any existing content. If the
281 // file does not already exist, creates it.
283 // On success, stores a pointer to the file in *result and returns OK. On
284 // failure stores nullptr in *result and returns non-OK.
286 // The returned file will only be accessed by one thread at a time.
287 virtual Status
ReopenWritableFile(const std::string
& /*fname*/,
288 std::unique_ptr
<WritableFile
>* /*result*/,
289 const EnvOptions
& /*options*/) {
290 return Status::NotSupported("Env::ReopenWritableFile() not supported.");
293 // Reuse an existing file by renaming it and opening it as writable.
294 virtual Status
ReuseWritableFile(const std::string
& fname
,
295 const std::string
& old_fname
,
296 std::unique_ptr
<WritableFile
>* result
,
297 const EnvOptions
& options
);
299 // Open `fname` for random read and write, if file doesn't exist the file
300 // will be created. On success, stores a pointer to the new file in
301 // *result and returns OK. On failure returns non-OK.
303 // The returned file will only be accessed by one thread at a time.
304 virtual Status
NewRandomRWFile(const std::string
& /*fname*/,
305 std::unique_ptr
<RandomRWFile
>* /*result*/,
306 const EnvOptions
& /*options*/) {
307 return Status::NotSupported("RandomRWFile is not implemented in this Env");
310 // Opens `fname` as a memory-mapped file for read and write (in-place updates
311 // only, i.e., no appends). On success, stores a raw buffer covering the whole
312 // file in `*result`. The file must exist prior to this call.
313 virtual Status
NewMemoryMappedFileBuffer(
314 const std::string
& /*fname*/,
315 std::unique_ptr
<MemoryMappedFileBuffer
>* /*result*/) {
316 return Status::NotSupported(
317 "MemoryMappedFileBuffer is not implemented in this Env");
320 // Create an object that represents a directory. Will fail if directory
321 // doesn't exist. If the directory exists, it will open the directory
322 // and create a new Directory object.
324 // On success, stores a pointer to the new Directory in
325 // *result and returns OK. On failure stores nullptr in *result and
327 virtual Status
NewDirectory(const std::string
& name
,
328 std::unique_ptr
<Directory
>* result
) = 0;
330 // Returns OK if the named file exists.
331 // NotFound if the named file does not exist,
332 // the calling process does not have permission to determine
333 // whether this file exists, or if the path is invalid.
334 // IOError if an IO Error was encountered
335 virtual Status
FileExists(const std::string
& fname
) = 0;
337 // Store in *result the names of the children of the specified directory.
338 // The names are relative to "dir", and shall never include the
339 // names `.` or `..`.
340 // Original contents of *results are dropped.
341 // Returns OK if "dir" exists and "*result" contains its children.
342 // NotFound if "dir" does not exist, the calling process does not have
343 // permission to access "dir", or if "dir" is invalid.
344 // IOError if an IO Error was encountered
345 virtual Status
GetChildren(const std::string
& dir
,
346 std::vector
<std::string
>* result
) = 0;
348 // Store in *result the attributes of the children of the specified directory.
349 // In case the implementation lists the directory prior to iterating the files
350 // and files are concurrently deleted, the deleted files will be omitted from
352 // The name attributes are relative to "dir", and shall never include the
353 // names `.` or `..`.
354 // Original contents of *results are dropped.
355 // Returns OK if "dir" exists and "*result" contains its children.
356 // NotFound if "dir" does not exist, the calling process does not have
357 // permission to access "dir", or if "dir" is invalid.
358 // IOError if an IO Error was encountered
359 virtual Status
GetChildrenFileAttributes(const std::string
& dir
,
360 std::vector
<FileAttributes
>* result
);
362 // Delete the named file.
363 virtual Status
DeleteFile(const std::string
& fname
) = 0;
365 // Truncate the named file to the specified size.
366 virtual Status
Truncate(const std::string
& /*fname*/, size_t /*size*/) {
367 return Status::NotSupported("Truncate is not supported for this Env");
370 // Create the specified directory. Returns error if directory exists.
371 virtual Status
CreateDir(const std::string
& dirname
) = 0;
373 // Creates directory if missing. Return Ok if it exists, or successful in
375 virtual Status
CreateDirIfMissing(const std::string
& dirname
) = 0;
377 // Delete the specified directory.
378 // Many implementations of this function will only delete a directory if it is
380 virtual Status
DeleteDir(const std::string
& dirname
) = 0;
382 // Store the size of fname in *file_size.
383 virtual Status
GetFileSize(const std::string
& fname
, uint64_t* file_size
) = 0;
385 // Store the last modification time of fname in *file_mtime.
386 virtual Status
GetFileModificationTime(const std::string
& fname
,
387 uint64_t* file_mtime
) = 0;
388 // Rename file src to target.
389 virtual Status
RenameFile(const std::string
& src
,
390 const std::string
& target
) = 0;
392 // Hard Link file src to target.
393 virtual Status
LinkFile(const std::string
& /*src*/,
394 const std::string
& /*target*/) {
395 return Status::NotSupported("LinkFile is not supported for this Env");
398 virtual Status
NumFileLinks(const std::string
& /*fname*/,
399 uint64_t* /*count*/) {
400 return Status::NotSupported(
401 "Getting number of file links is not supported for this Env");
404 virtual Status
AreFilesSame(const std::string
& /*first*/,
405 const std::string
& /*second*/, bool* /*res*/) {
406 return Status::NotSupported("AreFilesSame is not supported for this Env");
409 // Lock the specified file. Used to prevent concurrent access to
410 // the same db by multiple processes. On failure, stores nullptr in
411 // *lock and returns non-OK.
413 // On success, stores a pointer to the object that represents the
414 // acquired lock in *lock and returns OK. The caller should call
415 // UnlockFile(*lock) to release the lock. If the process exits,
416 // the lock will be automatically released.
418 // If somebody else already holds the lock, finishes immediately
419 // with a failure. I.e., this call does not wait for existing locks
422 // May create the named file if it does not already exist.
423 virtual Status
LockFile(const std::string
& fname
, FileLock
** lock
) = 0;
425 // Release the lock acquired by a previous successful call to LockFile.
426 // REQUIRES: lock was returned by a successful LockFile() call
427 // REQUIRES: lock has not already been unlocked.
428 virtual Status
UnlockFile(FileLock
* lock
) = 0;
430 // Opens `lib_name` as a dynamic library.
431 // If the 'search_path' is specified, breaks the path into its components
432 // based on the appropriate platform separator (";" or ";") and looks for the
433 // library in those directories. If 'search path is not specified, uses the
434 // default library path search mechanism (such as LD_LIBRARY_PATH). On
435 // success, stores a dynamic library in `*result`.
436 virtual Status
LoadLibrary(const std::string
& /*lib_name*/,
437 const std::string
& /*search_path */,
438 std::shared_ptr
<DynamicLibrary
>* /*result*/) {
439 return Status::NotSupported("LoadLibrary is not implemented in this Env");
442 // Priority for scheduling job in thread pool
443 enum Priority
{ BOTTOM
, LOW
, HIGH
, USER
, TOTAL
};
445 static std::string
PriorityToString(Priority priority
);
447 // Priority for requesting bytes in rate limiter scheduler
456 // Arrange to run "(*function)(arg)" once in a background thread, in
457 // the thread pool specified by pri. By default, jobs go to the 'LOW'
458 // priority thread pool.
460 // "function" may run in an unspecified thread. Multiple functions
461 // added to the same Env may run concurrently in different threads.
462 // I.e., the caller may not assume that background work items are
464 // When the UnSchedule function is called, the unschedFunction
465 // registered at the time of Schedule is invoked with arg as a parameter.
466 virtual void Schedule(void (*function
)(void* arg
), void* arg
,
467 Priority pri
= LOW
, void* tag
= nullptr,
468 void (*unschedFunction
)(void* arg
) = nullptr) = 0;
470 // Arrange to remove jobs for given arg from the queue_ if they are not
471 // already scheduled. Caller is expected to have exclusive lock on arg.
472 virtual int UnSchedule(void* /*arg*/, Priority
/*pri*/) { return 0; }
474 // Start a new thread, invoking "function(arg)" within the new thread.
475 // When "function(arg)" returns, the thread will be destroyed.
476 virtual void StartThread(void (*function
)(void* arg
), void* arg
) = 0;
478 // Start a new thread, invoking "function(args...)" within the new thread.
479 // When "function(args...)" returns, the thread will be destroyed.
480 template <typename FunctionT
, typename
... Args
>
481 void StartThreadTyped(FunctionT function
, Args
&&... args
) {
482 using FWType
= FunctorWrapper
<Args
...>;
485 auto* functor
= static_cast<FWType
*>(arg
);
489 new FWType(std::function
<void(Args
...)>(function
),
490 std::forward
<Args
>(args
)...));
493 // Wait for all threads started by StartThread to terminate.
494 virtual void WaitForJoin() {}
496 // Reserve available background threads in the specified thread pool.
497 virtual int ReserveThreads(int /*threads_to_be_reserved*/, Priority
/*pri*/) {
501 // Release a specific number of reserved threads from the specified thread
503 virtual int ReleaseThreads(int /*threads_to_be_released*/, Priority
/*pri*/) {
507 // Get thread pool queue length for specific thread pool.
508 virtual unsigned int GetThreadPoolQueueLen(Priority
/*pri*/ = LOW
) const {
512 // *path is set to a temporary directory that can be used for testing. It may
513 // or many not have just been created. The directory may or may not differ
514 // between runs of the same process, but subsequent calls will return the
516 virtual Status
GetTestDirectory(std::string
* path
) = 0;
518 // Create and returns a default logger (an instance of EnvLogger) for storing
519 // informational messages. Derived classes can override to provide custom
521 virtual Status
NewLogger(const std::string
& fname
,
522 std::shared_ptr
<Logger
>* result
);
524 // Returns the number of micro-seconds since some fixed point in time.
525 // It is often used as system time such as in GenericRateLimiter
526 // and other places so a port needs to return system time in order to work.
527 virtual uint64_t NowMicros() = 0;
529 // Returns the number of nano-seconds since some fixed point in time. Only
530 // useful for computing deltas of time in one run.
531 // Default implementation simply relies on NowMicros.
532 // In platform-specific implementations, NowNanos() should return time points
533 // that are MONOTONIC.
534 virtual uint64_t NowNanos() { return NowMicros() * 1000; }
536 // 0 indicates not supported.
537 virtual uint64_t NowCPUNanos() { return 0; }
539 // Sleep/delay the thread for the prescribed number of micro-seconds.
540 virtual void SleepForMicroseconds(int micros
) = 0;
542 // Get the current host name as a null terminated string iff the string
543 // length is < len. The hostname should otherwise be truncated to len.
544 virtual Status
GetHostName(char* name
, uint64_t len
) = 0;
546 // Get the current hostname from the given env as a std::string in result.
547 // The result may be truncated if the hostname is too
549 virtual Status
GetHostNameString(std::string
* result
);
551 // Get the number of seconds since the Epoch, 1970-01-01 00:00:00 (UTC).
552 // Only overwrites *unix_time on success.
553 virtual Status
GetCurrentTime(int64_t* unix_time
) = 0;
555 // Get full directory name for this db.
556 virtual Status
GetAbsolutePath(const std::string
& db_path
,
557 std::string
* output_path
) = 0;
559 // The number of background worker threads of a specific thread pool
560 // for this environment. 'LOW' is the default pool.
562 virtual void SetBackgroundThreads(int number
, Priority pri
= LOW
) = 0;
563 virtual int GetBackgroundThreads(Priority pri
= LOW
) = 0;
565 virtual Status
SetAllowNonOwnerAccess(bool /*allow_non_owner_access*/) {
566 return Status::NotSupported("Env::SetAllowNonOwnerAccess() not supported.");
569 // Enlarge number of background worker threads of a specific thread pool
570 // for this environment if it is smaller than specified. 'LOW' is the default
572 virtual void IncBackgroundThreadsIfNeeded(int number
, Priority pri
) = 0;
574 // Lower IO priority for threads from the specified pool.
575 virtual void LowerThreadPoolIOPriority(Priority
/*pool*/ = LOW
) {}
577 // Lower CPU priority for threads from the specified pool.
578 virtual Status
LowerThreadPoolCPUPriority(Priority
/*pool*/,
579 CpuPriority
/*pri*/) {
580 return Status::NotSupported(
581 "Env::LowerThreadPoolCPUPriority(Priority, CpuPriority) not supported");
584 // Lower CPU priority for threads from the specified pool.
585 virtual void LowerThreadPoolCPUPriority(Priority
/*pool*/ = LOW
) {}
587 // Converts seconds-since-Jan-01-1970 to a printable string
588 virtual std::string
TimeToString(uint64_t time
) = 0;
590 // Generates a human-readable unique ID that can be used to identify a DB.
591 // In built-in implementations, this is an RFC-4122 UUID string, but might
592 // not be in all implementations. Overriding is not recommended.
593 // NOTE: this has not be validated for use in cryptography
594 virtual std::string
GenerateUniqueId();
596 // OptimizeForLogWrite will create a new EnvOptions object that is a copy of
597 // the EnvOptions in the parameters, but is optimized for reading log files.
598 virtual EnvOptions
OptimizeForLogRead(const EnvOptions
& env_options
) const;
600 // OptimizeForManifestRead will create a new EnvOptions object that is a copy
601 // of the EnvOptions in the parameters, but is optimized for reading manifest
603 virtual EnvOptions
OptimizeForManifestRead(
604 const EnvOptions
& env_options
) const;
606 // OptimizeForLogWrite will create a new EnvOptions object that is a copy of
607 // the EnvOptions in the parameters, but is optimized for writing log files.
608 // Default implementation returns the copy of the same object.
609 virtual EnvOptions
OptimizeForLogWrite(const EnvOptions
& env_options
,
610 const DBOptions
& db_options
) const;
611 // OptimizeForManifestWrite will create a new EnvOptions object that is a copy
612 // of the EnvOptions in the parameters, but is optimized for writing manifest
613 // files. Default implementation returns the copy of the same object.
614 virtual EnvOptions
OptimizeForManifestWrite(
615 const EnvOptions
& env_options
) const;
617 // OptimizeForCompactionTableWrite will create a new EnvOptions object that is
618 // a copy of the EnvOptions in the parameters, but is optimized for writing
620 virtual EnvOptions
OptimizeForCompactionTableWrite(
621 const EnvOptions
& env_options
,
622 const ImmutableDBOptions
& immutable_ops
) const;
624 // OptimizeForCompactionTableWrite will create a new EnvOptions object that
625 // is a copy of the EnvOptions in the parameters, but is optimized for reading
627 virtual EnvOptions
OptimizeForCompactionTableRead(
628 const EnvOptions
& env_options
,
629 const ImmutableDBOptions
& db_options
) const;
631 // OptimizeForBlobFileRead will create a new EnvOptions object that
632 // is a copy of the EnvOptions in the parameters, but is optimized for reading
634 virtual EnvOptions
OptimizeForBlobFileRead(
635 const EnvOptions
& env_options
,
636 const ImmutableDBOptions
& db_options
) const;
638 // Returns the status of all threads that belong to the current Env.
639 virtual Status
GetThreadList(std::vector
<ThreadStatus
>* /*thread_list*/) {
640 return Status::NotSupported("Env::GetThreadList() not supported.");
643 // Returns the pointer to ThreadStatusUpdater. This function will be
644 // used in RocksDB internally to update thread status and supports
646 virtual ThreadStatusUpdater
* GetThreadStatusUpdater() const {
647 return thread_status_updater_
;
650 // Returns the ID of the current thread.
651 virtual uint64_t GetThreadID() const;
653 // This seems to clash with a macro on Windows, so #undef it here
656 // Get the amount of free disk space
657 virtual Status
GetFreeSpace(const std::string
& /*path*/,
658 uint64_t* /*diskfree*/) {
659 return Status::NotSupported("Env::GetFreeSpace() not supported.");
662 // Check whether the specified path is a directory
663 virtual Status
IsDirectory(const std::string
& /*path*/, bool* /*is_dir*/) {
664 return Status::NotSupported("Env::IsDirectory() not supported.");
667 virtual void SanitizeEnvOptions(EnvOptions
* /*env_opts*/) const {}
669 // Get the FileSystem implementation this Env was constructed with. It
670 // could be a fully implemented one, or a wrapper class around the Env
671 const std::shared_ptr
<FileSystem
>& GetFileSystem() const;
673 // Get the SystemClock implementation this Env was constructed with. It
674 // could be a fully implemented one, or a wrapper class around the Env
675 const std::shared_ptr
<SystemClock
>& GetSystemClock() const;
677 // If you're adding methods here, remember to add them to EnvWrapper too.
680 // The pointer to an internal structure that will update the
681 // status of each thread.
682 ThreadStatusUpdater
* thread_status_updater_
;
684 // Pointer to the underlying FileSystem implementation
685 std::shared_ptr
<FileSystem
> file_system_
;
687 // Pointer to the underlying SystemClock implementation
688 std::shared_ptr
<SystemClock
> system_clock_
;
691 static const size_t kMaxHostNameLen
= 256;
694 // The factory function to construct a ThreadStatusUpdater. Any Env
695 // that supports GetThreadList() feature should call this function in its
696 // constructor to initialize thread_status_updater_.
697 ThreadStatusUpdater
* CreateThreadStatusUpdater();
699 // A file abstraction for reading sequentially through a file
700 class SequentialFile
{
703 virtual ~SequentialFile();
705 // Read up to "n" bytes from the file. "scratch[0..n-1]" may be
706 // written by this routine. Sets "*result" to the data that was
707 // read (including if fewer than "n" bytes were successfully read).
708 // May set "*result" to point at data in "scratch[0..n-1]", so
709 // "scratch[0..n-1]" must be live when "*result" is used.
710 // If an error was encountered, returns a non-OK status.
712 // After call, result->size() < n only if end of file has been
713 // reached (or non-OK status). Read might fail if called again after
714 // first result->size() < n.
716 // REQUIRES: External synchronization
717 virtual Status
Read(size_t n
, Slice
* result
, char* scratch
) = 0;
719 // Skip "n" bytes from the file. This is guaranteed to be no
720 // slower that reading the same data, but may be faster.
722 // If end of file is reached, skipping will stop at the end of the
723 // file, and Skip will return OK.
725 // REQUIRES: External synchronization
726 virtual Status
Skip(uint64_t n
) = 0;
728 // Indicates the upper layers if the current SequentialFile implementation
730 virtual bool use_direct_io() const { return false; }
732 // Use the returned alignment value to allocate
733 // aligned buffer for Direct I/O
734 virtual size_t GetRequiredBufferAlignment() const { return kDefaultPageSize
; }
736 // Remove any kind of caching of data from the offset to offset+length
737 // of this file. If the length is 0, then it refers to the end of file.
738 // If the system is not caching the file contents, then this is a noop.
739 virtual Status
InvalidateCache(size_t /*offset*/, size_t /*length*/) {
740 return Status::NotSupported(
741 "SequentialFile::InvalidateCache not supported.");
744 // Positioned Read for direct I/O
745 // If Direct I/O enabled, offset, n, and scratch should be properly aligned
746 virtual Status
PositionedRead(uint64_t /*offset*/, size_t /*n*/,
747 Slice
* /*result*/, char* /*scratch*/) {
748 return Status::NotSupported(
749 "SequentialFile::PositionedRead() not supported.");
752 // If you're adding methods here, remember to add them to
753 // SequentialFileWrapper too.
756 // A read IO request structure for use in MultiRead
758 // File offset in bytes
761 // Length to read in bytes. `result` only returns fewer bytes if end of file
762 // is hit (or `status` is not OK).
765 // A buffer that MultiRead() can optionally place data in. It can
766 // ignore this and allocate its own buffer
769 // Output parameter set by MultiRead() to point to the data buffer, and
770 // the number of valid bytes
777 // A file abstraction for randomly reading the contents of a file.
778 class RandomAccessFile
{
780 RandomAccessFile() {}
781 virtual ~RandomAccessFile();
783 // Read up to "n" bytes from the file starting at "offset".
784 // "scratch[0..n-1]" may be written by this routine. Sets "*result"
785 // to the data that was read (including if fewer than "n" bytes were
786 // successfully read). May set "*result" to point at data in
787 // "scratch[0..n-1]", so "scratch[0..n-1]" must be live when
788 // "*result" is used. If an error was encountered, returns a non-OK
791 // After call, result->size() < n only if end of file has been
792 // reached (or non-OK status). Read might fail if called again after
793 // first result->size() < n.
795 // Safe for concurrent use by multiple threads.
796 // If Direct I/O enabled, offset, n, and scratch should be aligned properly.
797 virtual Status
Read(uint64_t offset
, size_t n
, Slice
* result
,
798 char* scratch
) const = 0;
800 // Readahead the file starting from offset by n bytes for caching.
801 virtual Status
Prefetch(uint64_t /*offset*/, size_t /*n*/) {
805 // Read a bunch of blocks as described by reqs. The blocks can
806 // optionally be read in parallel. This is a synchronous call, i.e it
807 // should return after all reads have completed. The reads will be
808 // non-overlapping. If the function return Status is not ok, status of
809 // individual requests will be ignored and return status will be assumed
810 // for all read requests. The function return status is only meant for
811 // any errors that occur before even processing specific read requests
812 virtual Status
MultiRead(ReadRequest
* reqs
, size_t num_reqs
) {
813 assert(reqs
!= nullptr);
814 for (size_t i
= 0; i
< num_reqs
; ++i
) {
815 ReadRequest
& req
= reqs
[i
];
816 req
.status
= Read(req
.offset
, req
.len
, &req
.result
, req
.scratch
);
821 // Tries to get an unique ID for this file that will be the same each time
822 // the file is opened (and will stay the same while the file is open).
823 // Furthermore, it tries to make this ID at most "max_size" bytes. If such an
824 // ID can be created this function returns the length of the ID and places it
825 // in "id"; otherwise, this function returns 0, in which case "id"
826 // may not have been modified.
828 // This function guarantees, for IDs from a given environment, two unique ids
829 // cannot be made equal to each other by adding arbitrary bytes to one of
830 // them. That is, no unique ID is the prefix of another.
832 // This function guarantees that the returned ID will not be interpretable as
835 // Note: these IDs are only valid for the duration of the process.
836 virtual size_t GetUniqueId(char* /*id*/, size_t /*max_size*/) const {
837 return 0; // Default implementation to prevent issues with backwards
841 enum AccessPattern
{ NORMAL
, RANDOM
, SEQUENTIAL
, WILLNEED
, DONTNEED
};
843 virtual void Hint(AccessPattern
/*pattern*/) {}
845 // Indicates the upper layers if the current RandomAccessFile implementation
847 virtual bool use_direct_io() const { return false; }
849 // Use the returned alignment value to allocate
850 // aligned buffer for Direct I/O
851 virtual size_t GetRequiredBufferAlignment() const { return kDefaultPageSize
; }
853 // Remove any kind of caching of data from the offset to offset+length
854 // of this file. If the length is 0, then it refers to the end of file.
855 // If the system is not caching the file contents, then this is a noop.
856 virtual Status
InvalidateCache(size_t /*offset*/, size_t /*length*/) {
857 return Status::NotSupported(
858 "RandomAccessFile::InvalidateCache not supported.");
861 // If you're adding methods here, remember to add them to
862 // RandomAccessFileWrapper too.
865 // A file abstraction for sequential writing. The implementation
866 // must provide buffering since callers may append small fragments
867 // at a time to the file.
871 : last_preallocated_block_(0),
872 preallocation_block_size_(0),
873 io_priority_(Env::IO_TOTAL
),
874 write_hint_(Env::WLTH_NOT_SET
),
875 strict_bytes_per_sync_(false) {}
877 explicit WritableFile(const EnvOptions
& options
)
878 : last_preallocated_block_(0),
879 preallocation_block_size_(0),
880 io_priority_(Env::IO_TOTAL
),
881 write_hint_(Env::WLTH_NOT_SET
),
882 strict_bytes_per_sync_(options
.strict_bytes_per_sync
) {}
883 // No copying allowed
884 WritableFile(const WritableFile
&) = delete;
885 void operator=(const WritableFile
&) = delete;
887 virtual ~WritableFile();
889 // Append data to the end of the file
890 // Note: A WritableFile object must support either Append or
891 // PositionedAppend, so the users cannot mix the two.
892 virtual Status
Append(const Slice
& data
) = 0;
894 // Append data with verification information.
895 // Note that this API change is experimental and it might be changed in
896 // the future. Currently, RocksDB only generates crc32c based checksum for
897 // the file writes when the checksum handoff option is set.
898 // Expected behavior: if currently ChecksumType::kCRC32C is not supported by
899 // WritableFile, the information in DataVerificationInfo can be ignored
900 // (i.e. does not perform checksum verification).
901 virtual Status
Append(const Slice
& data
,
902 const DataVerificationInfo
& /* verification_info */) {
906 // PositionedAppend data to the specified offset. The new EOF after append
907 // must be larger than the previous EOF. This is to be used when writes are
908 // not backed by OS buffers and hence has to always start from the start of
909 // the sector. The implementation thus needs to also rewrite the last
911 // Note: PositionAppend does not guarantee moving the file offset after the
912 // write. A WritableFile object must support either Append or
913 // PositionedAppend, so the users cannot mix the two.
915 // PositionedAppend() can only happen on the page/sector boundaries. For that
916 // reason, if the last write was an incomplete sector we still need to rewind
917 // back to the nearest sector/page and rewrite the portion of it with whatever
918 // we need to add. We need to keep where we stop writing.
920 // PositionedAppend() can only write whole sectors. For that reason we have to
921 // pad with zeros for the last write and trim the file when closing according
922 // to the position we keep in the previous step.
924 // PositionedAppend() requires aligned buffer to be passed in. The alignment
925 // required is queried via GetRequiredBufferAlignment()
926 virtual Status
PositionedAppend(const Slice
& /* data */,
927 uint64_t /* offset */) {
928 return Status::NotSupported(
929 "WritableFile::PositionedAppend() not supported.");
932 // PositionedAppend data with verification information.
933 // Note that this API change is experimental and it might be changed in
934 // the future. Currently, RocksDB only generates crc32c based checksum for
935 // the file writes when the checksum handoff option is set.
936 // Expected behavior: if currently ChecksumType::kCRC32C is not supported by
937 // WritableFile, the information in DataVerificationInfo can be ignored
938 // (i.e. does not perform checksum verification).
939 virtual Status
PositionedAppend(
940 const Slice
& /* data */, uint64_t /* offset */,
941 const DataVerificationInfo
& /* verification_info */) {
942 return Status::NotSupported("PositionedAppend");
945 // Truncate is necessary to trim the file to the correct size
946 // before closing. It is not always possible to keep track of the file
947 // size due to whole pages writes. The behavior is undefined if called
948 // with other writes to follow.
949 virtual Status
Truncate(uint64_t /*size*/) { return Status::OK(); }
950 virtual Status
Close() = 0;
951 virtual Status
Flush() = 0;
952 virtual Status
Sync() = 0; // sync data
955 * Sync data and/or metadata as well.
956 * By default, sync only data.
957 * Override this method for environments where we need to sync
960 virtual Status
Fsync() { return Sync(); }
962 // true if Sync() and Fsync() are safe to call concurrently with Append()
964 virtual bool IsSyncThreadSafe() const { return false; }
966 // Indicates the upper layers if the current WritableFile implementation
968 virtual bool use_direct_io() const { return false; }
970 // Use the returned alignment value to allocate
971 // aligned buffer for Direct I/O
972 virtual size_t GetRequiredBufferAlignment() const { return kDefaultPageSize
; }
975 * If rate limiting is enabled, change the file-granularity priority used in
976 * rate-limiting writes.
978 * In the presence of finer-granularity priority such as
979 * `WriteOptions::rate_limiter_priority`, this file-granularity priority may
980 * be overridden by a non-Env::IO_TOTAL finer-granularity priority and used as
981 * a fallback for Env::IO_TOTAL finer-granularity priority.
983 * If rate limiting is not enabled, this call has no effect.
985 virtual void SetIOPriority(Env::IOPriority pri
) { io_priority_
= pri
; }
987 virtual Env::IOPriority
GetIOPriority() { return io_priority_
; }
989 virtual void SetWriteLifeTimeHint(Env::WriteLifeTimeHint hint
) {
993 virtual Env::WriteLifeTimeHint
GetWriteLifeTimeHint() { return write_hint_
; }
995 * Get the size of valid data in the file.
997 virtual uint64_t GetFileSize() { return 0; }
1000 * Get and set the default pre-allocation block size for writes to
1001 * this file. If non-zero, then Allocate will be used to extend the
1002 * underlying storage of a file (generally via fallocate) if the Env
1003 * instance supports it.
1005 virtual void SetPreallocationBlockSize(size_t size
) {
1006 preallocation_block_size_
= size
;
1009 virtual void GetPreallocationStatus(size_t* block_size
,
1010 size_t* last_allocated_block
) {
1011 *last_allocated_block
= last_preallocated_block_
;
1012 *block_size
= preallocation_block_size_
;
1015 // For documentation, refer to RandomAccessFile::GetUniqueId()
1016 virtual size_t GetUniqueId(char* /*id*/, size_t /*max_size*/) const {
1017 return 0; // Default implementation to prevent issues with backwards
1020 // Remove any kind of caching of data from the offset to offset+length
1021 // of this file. If the length is 0, then it refers to the end of file.
1022 // If the system is not caching the file contents, then this is a noop.
1023 // This call has no effect on dirty pages in the cache.
1024 virtual Status
InvalidateCache(size_t /*offset*/, size_t /*length*/) {
1025 return Status::NotSupported("WritableFile::InvalidateCache not supported.");
1028 // Sync a file range with disk.
1029 // offset is the starting byte of the file range to be synchronized.
1030 // nbytes specifies the length of the range to be synchronized.
1031 // This asks the OS to initiate flushing the cached data to disk,
1032 // without waiting for completion.
1033 // Default implementation does nothing.
1034 virtual Status
RangeSync(uint64_t /*offset*/, uint64_t /*nbytes*/) {
1035 if (strict_bytes_per_sync_
) {
1038 return Status::OK();
1041 // PrepareWrite performs any necessary preparation for a write
1042 // before the write actually occurs. This allows for pre-allocation
1043 // of space on devices where it can result in less file
1044 // fragmentation and/or less waste from over-zealous filesystem
1046 virtual void PrepareWrite(size_t offset
, size_t len
) {
1047 if (preallocation_block_size_
== 0) {
1050 // If this write would cross one or more preallocation blocks,
1051 // determine what the last preallocation block necessary to
1052 // cover this write would be and Allocate to that point.
1053 const auto block_size
= preallocation_block_size_
;
1054 size_t new_last_preallocated_block
=
1055 (offset
+ len
+ block_size
- 1) / block_size
;
1056 if (new_last_preallocated_block
> last_preallocated_block_
) {
1057 size_t num_spanned_blocks
=
1058 new_last_preallocated_block
- last_preallocated_block_
;
1059 // TODO: Don't ignore errors from allocate
1060 Allocate(block_size
* last_preallocated_block_
,
1061 block_size
* num_spanned_blocks
)
1062 .PermitUncheckedError();
1063 last_preallocated_block_
= new_last_preallocated_block
;
1067 // Pre-allocates space for a file.
1068 virtual Status
Allocate(uint64_t /*offset*/, uint64_t /*len*/) {
1069 return Status::OK();
1072 // If you're adding methods here, remember to add them to
1073 // WritableFileWrapper too.
1076 size_t preallocation_block_size() { return preallocation_block_size_
; }
1079 size_t last_preallocated_block_
;
1080 size_t preallocation_block_size_
;
1083 Env::IOPriority io_priority_
;
1084 Env::WriteLifeTimeHint write_hint_
;
1085 const bool strict_bytes_per_sync_
;
1088 // A file abstraction for random reading and writing.
1089 class RandomRWFile
{
1092 // No copying allowed
1093 RandomRWFile(const RandomRWFile
&) = delete;
1094 RandomRWFile
& operator=(const RandomRWFile
&) = delete;
1096 virtual ~RandomRWFile() {}
1098 // Indicates if the class makes use of direct I/O
1099 // If false you must pass aligned buffer to Write()
1100 virtual bool use_direct_io() const { return false; }
1102 // Use the returned alignment value to allocate
1103 // aligned buffer for Direct I/O
1104 virtual size_t GetRequiredBufferAlignment() const { return kDefaultPageSize
; }
1106 // Write bytes in `data` at offset `offset`, Returns Status::OK() on success.
1107 // Pass aligned buffer when use_direct_io() returns true.
1108 virtual Status
Write(uint64_t offset
, const Slice
& data
) = 0;
1110 // Read up to `n` bytes starting from offset `offset` and store them in
1111 // result, provided `scratch` size should be at least `n`.
1113 // After call, result->size() < n only if end of file has been
1114 // reached (or non-OK status). Read might fail if called again after
1115 // first result->size() < n.
1117 // Returns Status::OK() on success.
1118 virtual Status
Read(uint64_t offset
, size_t n
, Slice
* result
,
1119 char* scratch
) const = 0;
1121 virtual Status
Flush() = 0;
1123 virtual Status
Sync() = 0;
1125 virtual Status
Fsync() { return Sync(); }
1127 virtual Status
Close() = 0;
1129 // If you're adding methods here, remember to add them to
1130 // RandomRWFileWrapper too.
1133 // MemoryMappedFileBuffer object represents a memory-mapped file's raw buffer.
1134 // Subclasses should release the mapping upon destruction.
1135 class MemoryMappedFileBuffer
{
1137 MemoryMappedFileBuffer(void* _base
, size_t _length
)
1138 : base_(_base
), length_(_length
) {}
1140 virtual ~MemoryMappedFileBuffer() = 0;
1142 // We do not want to unmap this twice. We can make this class
1143 // movable if desired, however, since
1144 MemoryMappedFileBuffer(const MemoryMappedFileBuffer
&) = delete;
1145 MemoryMappedFileBuffer
& operator=(const MemoryMappedFileBuffer
&) = delete;
1147 void* GetBase() const { return base_
; }
1148 size_t GetLen() const { return length_
; }
1152 const size_t length_
;
1155 // Directory object represents collection of files and implements
1156 // filesystem operations that can be executed on directories.
1159 virtual ~Directory() {}
1160 // Fsync directory. Can be called concurrently from multiple threads.
1161 virtual Status
Fsync() = 0;
1163 virtual Status
Close() { return Status::NotSupported("Close"); }
1165 virtual size_t GetUniqueId(char* /*id*/, size_t /*max_size*/) const {
1169 // If you're adding methods here, remember to add them to
1170 // DirectoryWrapper too.
1173 enum InfoLogLevel
: unsigned char {
1180 NUM_INFO_LOG_LEVELS
,
1183 // An interface for writing log messages.
1185 // Exceptions MUST NOT propagate out of overridden functions into RocksDB,
1186 // because RocksDB is not exception-safe. This could cause undefined behavior
1187 // including data loss, unreported corruption, deadlocks, and more.
1190 size_t kDoNotSupportGetLogFileSize
= (std::numeric_limits
<size_t>::max
)();
1192 explicit Logger(const InfoLogLevel log_level
= InfoLogLevel::INFO_LEVEL
)
1193 : closed_(false), log_level_(log_level
) {}
1194 // No copying allowed
1195 Logger(const Logger
&) = delete;
1196 void operator=(const Logger
&) = delete;
1200 // Close the log file. Must be called before destructor. If the return
1201 // status is NotSupported(), it means the implementation does cleanup in
1203 virtual Status
Close();
1205 // Write a header to the log file with the specified format
1206 // It is recommended that you log all header information at the start of the
1207 // application. But it is not enforced.
1208 virtual void LogHeader(const char* format
, va_list ap
) {
1209 // Default implementation does a simple INFO level log write.
1210 // Please override as per the logger class requirement.
1211 Logv(InfoLogLevel::INFO_LEVEL
, format
, ap
);
1214 // Write an entry to the log file with the specified format.
1216 // Users who override the `Logv()` overload taking `InfoLogLevel` do not need
1217 // to implement this, unless they explicitly invoke it in
1218 // `Logv(InfoLogLevel, ...)`.
1219 virtual void Logv(const char* /* format */, va_list /* ap */) {
1223 // Write an entry to the log file with the specified log level
1224 // and format. Any log with level under the internal log level
1225 // of *this (see @SetInfoLogLevel and @GetInfoLogLevel) will not be
1227 virtual void Logv(const InfoLogLevel log_level
, const char* format
,
1230 virtual size_t GetLogFileSize() const { return kDoNotSupportGetLogFileSize
; }
1231 // Flush to the OS buffers
1232 virtual void Flush() {}
1233 virtual InfoLogLevel
GetInfoLogLevel() const { return log_level_
; }
1234 virtual void SetInfoLogLevel(const InfoLogLevel log_level
) {
1235 log_level_
= log_level
;
1238 // If you're adding methods here, remember to add them to LoggerWrapper too.
1241 virtual Status
CloseImpl();
1245 InfoLogLevel log_level_
;
1248 // Identifies a locked file. Except in custom Env/Filesystem implementations,
1249 // the lifetime of a FileLock object should be managed only by LockFile() and
1254 virtual ~FileLock();
1257 // No copying allowed
1258 FileLock(const FileLock
&) = delete;
1259 void operator=(const FileLock
&) = delete;
1262 class DynamicLibrary
{
1264 virtual ~DynamicLibrary() {}
1266 // Returns the name of the dynamic library.
1267 virtual const char* Name() const = 0;
1269 // Loads the symbol for sym_name from the library and updates the input
1270 // function. Returns the loaded symbol.
1271 template <typename T
>
1272 Status
LoadFunction(const std::string
& sym_name
, std::function
<T
>* function
) {
1273 assert(nullptr != function
);
1274 void* ptr
= nullptr;
1275 Status s
= LoadSymbol(sym_name
, &ptr
);
1276 *function
= reinterpret_cast<T
*>(ptr
);
1279 // Loads and returns the symbol for sym_name from the library.
1280 virtual Status
LoadSymbol(const std::string
& sym_name
, void** func
) = 0;
1283 extern void LogFlush(const std::shared_ptr
<Logger
>& info_log
);
1285 extern void Log(const InfoLogLevel log_level
,
1286 const std::shared_ptr
<Logger
>& info_log
, const char* format
,
1287 ...) ROCKSDB_PRINTF_FORMAT_ATTR(3, 4);
1289 // a set of log functions with different log levels.
1290 extern void Header(const std::shared_ptr
<Logger
>& info_log
, const char* format
,
1291 ...) ROCKSDB_PRINTF_FORMAT_ATTR(2, 3);
1292 extern void Debug(const std::shared_ptr
<Logger
>& info_log
, const char* format
,
1293 ...) ROCKSDB_PRINTF_FORMAT_ATTR(2, 3);
1294 extern void Info(const std::shared_ptr
<Logger
>& info_log
, const char* format
,
1295 ...) ROCKSDB_PRINTF_FORMAT_ATTR(2, 3);
1296 extern void Warn(const std::shared_ptr
<Logger
>& info_log
, const char* format
,
1297 ...) ROCKSDB_PRINTF_FORMAT_ATTR(2, 3);
1298 extern void Error(const std::shared_ptr
<Logger
>& info_log
, const char* format
,
1299 ...) ROCKSDB_PRINTF_FORMAT_ATTR(2, 3);
1300 extern void Fatal(const std::shared_ptr
<Logger
>& info_log
, const char* format
,
1301 ...) ROCKSDB_PRINTF_FORMAT_ATTR(2, 3);
1303 // Log the specified data to *info_log if info_log is non-nullptr.
1304 // The default info log level is InfoLogLevel::INFO_LEVEL.
1305 extern void Log(const std::shared_ptr
<Logger
>& info_log
, const char* format
,
1306 ...) ROCKSDB_PRINTF_FORMAT_ATTR(2, 3);
1308 extern void LogFlush(Logger
* info_log
);
1310 extern void Log(const InfoLogLevel log_level
, Logger
* info_log
,
1311 const char* format
, ...) ROCKSDB_PRINTF_FORMAT_ATTR(3, 4);
1313 // The default info log level is InfoLogLevel::INFO_LEVEL.
1314 extern void Log(Logger
* info_log
, const char* format
, ...)
1315 ROCKSDB_PRINTF_FORMAT_ATTR(2, 3);
1317 // a set of log functions with different log levels.
1318 extern void Header(Logger
* info_log
, const char* format
, ...)
1319 ROCKSDB_PRINTF_FORMAT_ATTR(2, 3);
1320 extern void Debug(Logger
* info_log
, const char* format
, ...)
1321 ROCKSDB_PRINTF_FORMAT_ATTR(2, 3);
1322 extern void Info(Logger
* info_log
, const char* format
, ...)
1323 ROCKSDB_PRINTF_FORMAT_ATTR(2, 3);
1324 extern void Warn(Logger
* info_log
, const char* format
, ...)
1325 ROCKSDB_PRINTF_FORMAT_ATTR(2, 3);
1326 extern void Error(Logger
* info_log
, const char* format
, ...)
1327 ROCKSDB_PRINTF_FORMAT_ATTR(2, 3);
1328 extern void Fatal(Logger
* info_log
, const char* format
, ...)
1329 ROCKSDB_PRINTF_FORMAT_ATTR(2, 3);
1331 // A utility routine: write "data" to the named file.
1332 extern Status
WriteStringToFile(Env
* env
, const Slice
& data
,
1333 const std::string
& fname
,
1334 bool should_sync
= false);
1336 // A utility routine: read contents of named file into *data
1337 extern Status
ReadFileToString(Env
* env
, const std::string
& fname
,
1340 // Below are helpers for wrapping most of the classes in this file.
1341 // They forward all calls to another instance of the class.
1342 // Useful when wrapping the default implementations.
1343 // Typical usage is to inherit your wrapper from *Wrapper, e.g.:
1345 // class MySequentialFileWrapper : public
1346 // ROCKSDB_NAMESPACE::SequentialFileWrapper {
1348 // MySequentialFileWrapper(ROCKSDB_NAMESPACE::SequentialFile* target):
1349 // ROCKSDB_NAMESPACE::SequentialFileWrapper(target) {}
1350 // Status Read(size_t n, Slice* result, char* scratch) override {
1351 // cout << "Doing a read of size " << n << "!" << endl;
1352 // return ROCKSDB_NAMESPACE::SequentialFileWrapper::Read(n, result,
1355 // // All other methods are forwarded to target_ automatically.
1358 // This is often more convenient than inheriting the class directly because
1359 // (a) Don't have to override and forward all methods - the Wrapper will
1360 // forward everything you're not explicitly overriding.
1361 // (b) Don't need to update the wrapper when more methods are added to the
1362 // rocksdb class. Unless you actually want to override the behavior.
1363 // (And unless rocksdb people forgot to update the *Wrapper class.)
1365 // An implementation of Env that forwards all calls to another Env.
1366 // May be useful to clients who wish to override just part of the
1367 // functionality of another Env.
1368 class EnvWrapper
: public Env
{
1370 // The Target struct allows an Env to be stored as a raw (Env*) or
1371 // std::shared_ptr<Env>. By using this struct, the wrapping/calling
1372 // class does not need to worry about the ownership/lifetime of the
1373 // wrapped target env. If the guard is set, then the Env will point
1374 // to the guard.get().
1376 Env
* env
; // The raw Env
1377 std::shared_ptr
<Env
> guard
; // The guarded Env
1379 // Creates a Target without assuming ownership of the target Env
1380 explicit Target(Env
* t
) : env(t
) {}
1382 // Creates a Target from the guarded env, assuming ownership
1383 explicit Target(std::unique_ptr
<Env
>&& t
) : guard(t
.release()) {
1387 // Creates a Target from the guarded env, assuming ownership
1388 explicit Target(const std::shared_ptr
<Env
>& t
) : guard(t
) {
1392 // Makes sure the raw Env is not nullptr
1394 if (guard
.get() != nullptr) {
1396 } else if (env
== nullptr) {
1397 env
= Env::Default();
1402 // Initialize an EnvWrapper that delegates all calls to *t
1403 explicit EnvWrapper(Env
* t
);
1404 explicit EnvWrapper(std::unique_ptr
<Env
>&& t
);
1405 explicit EnvWrapper(const std::shared_ptr
<Env
>& t
);
1406 ~EnvWrapper() override
;
1408 // Return the target to which this Env forwards all calls
1409 Env
* target() const { return target_
.env
; }
1411 // Deprecated. Will be removed in a major release. Derived classes
1412 // should implement this method.
1413 const char* Name() const override
{ return target_
.env
->Name(); }
1415 // The following text is boilerplate that forwards all methods to target()
1416 Status
RegisterDbPaths(const std::vector
<std::string
>& paths
) override
{
1417 return target_
.env
->RegisterDbPaths(paths
);
1420 Status
UnregisterDbPaths(const std::vector
<std::string
>& paths
) override
{
1421 return target_
.env
->UnregisterDbPaths(paths
);
1424 Status
NewSequentialFile(const std::string
& f
,
1425 std::unique_ptr
<SequentialFile
>* r
,
1426 const EnvOptions
& options
) override
{
1427 return target_
.env
->NewSequentialFile(f
, r
, options
);
1429 Status
NewRandomAccessFile(const std::string
& f
,
1430 std::unique_ptr
<RandomAccessFile
>* r
,
1431 const EnvOptions
& options
) override
{
1432 return target_
.env
->NewRandomAccessFile(f
, r
, options
);
1434 Status
NewWritableFile(const std::string
& f
, std::unique_ptr
<WritableFile
>* r
,
1435 const EnvOptions
& options
) override
{
1436 return target_
.env
->NewWritableFile(f
, r
, options
);
1438 Status
ReopenWritableFile(const std::string
& fname
,
1439 std::unique_ptr
<WritableFile
>* result
,
1440 const EnvOptions
& options
) override
{
1441 return target_
.env
->ReopenWritableFile(fname
, result
, options
);
1443 Status
ReuseWritableFile(const std::string
& fname
,
1444 const std::string
& old_fname
,
1445 std::unique_ptr
<WritableFile
>* r
,
1446 const EnvOptions
& options
) override
{
1447 return target_
.env
->ReuseWritableFile(fname
, old_fname
, r
, options
);
1449 Status
NewRandomRWFile(const std::string
& fname
,
1450 std::unique_ptr
<RandomRWFile
>* result
,
1451 const EnvOptions
& options
) override
{
1452 return target_
.env
->NewRandomRWFile(fname
, result
, options
);
1454 Status
NewMemoryMappedFileBuffer(
1455 const std::string
& fname
,
1456 std::unique_ptr
<MemoryMappedFileBuffer
>* result
) override
{
1457 return target_
.env
->NewMemoryMappedFileBuffer(fname
, result
);
1459 Status
NewDirectory(const std::string
& name
,
1460 std::unique_ptr
<Directory
>* result
) override
{
1461 return target_
.env
->NewDirectory(name
, result
);
1463 Status
FileExists(const std::string
& f
) override
{
1464 return target_
.env
->FileExists(f
);
1466 Status
GetChildren(const std::string
& dir
,
1467 std::vector
<std::string
>* r
) override
{
1468 return target_
.env
->GetChildren(dir
, r
);
1470 Status
GetChildrenFileAttributes(
1471 const std::string
& dir
, std::vector
<FileAttributes
>* result
) override
{
1472 return target_
.env
->GetChildrenFileAttributes(dir
, result
);
1474 Status
DeleteFile(const std::string
& f
) override
{
1475 return target_
.env
->DeleteFile(f
);
1477 Status
Truncate(const std::string
& fname
, size_t size
) override
{
1478 return target_
.env
->Truncate(fname
, size
);
1480 Status
CreateDir(const std::string
& d
) override
{
1481 return target_
.env
->CreateDir(d
);
1483 Status
CreateDirIfMissing(const std::string
& d
) override
{
1484 return target_
.env
->CreateDirIfMissing(d
);
1486 Status
DeleteDir(const std::string
& d
) override
{
1487 return target_
.env
->DeleteDir(d
);
1489 Status
GetFileSize(const std::string
& f
, uint64_t* s
) override
{
1490 return target_
.env
->GetFileSize(f
, s
);
1493 Status
GetFileModificationTime(const std::string
& fname
,
1494 uint64_t* file_mtime
) override
{
1495 return target_
.env
->GetFileModificationTime(fname
, file_mtime
);
1498 Status
RenameFile(const std::string
& s
, const std::string
& t
) override
{
1499 return target_
.env
->RenameFile(s
, t
);
1502 Status
LinkFile(const std::string
& s
, const std::string
& t
) override
{
1503 return target_
.env
->LinkFile(s
, t
);
1506 Status
NumFileLinks(const std::string
& fname
, uint64_t* count
) override
{
1507 return target_
.env
->NumFileLinks(fname
, count
);
1510 Status
AreFilesSame(const std::string
& first
, const std::string
& second
,
1511 bool* res
) override
{
1512 return target_
.env
->AreFilesSame(first
, second
, res
);
1515 Status
LockFile(const std::string
& f
, FileLock
** l
) override
{
1516 return target_
.env
->LockFile(f
, l
);
1519 Status
UnlockFile(FileLock
* l
) override
{ return target_
.env
->UnlockFile(l
); }
1521 Status
IsDirectory(const std::string
& path
, bool* is_dir
) override
{
1522 return target_
.env
->IsDirectory(path
, is_dir
);
1525 Status
LoadLibrary(const std::string
& lib_name
,
1526 const std::string
& search_path
,
1527 std::shared_ptr
<DynamicLibrary
>* result
) override
{
1528 return target_
.env
->LoadLibrary(lib_name
, search_path
, result
);
1531 void Schedule(void (*f
)(void* arg
), void* a
, Priority pri
,
1532 void* tag
= nullptr, void (*u
)(void* arg
) = nullptr) override
{
1533 return target_
.env
->Schedule(f
, a
, pri
, tag
, u
);
1536 int UnSchedule(void* tag
, Priority pri
) override
{
1537 return target_
.env
->UnSchedule(tag
, pri
);
1540 void StartThread(void (*f
)(void*), void* a
) override
{
1541 return target_
.env
->StartThread(f
, a
);
1543 void WaitForJoin() override
{ return target_
.env
->WaitForJoin(); }
1544 unsigned int GetThreadPoolQueueLen(Priority pri
= LOW
) const override
{
1545 return target_
.env
->GetThreadPoolQueueLen(pri
);
1548 int ReserveThreads(int threads_to_be_reserved
, Priority pri
) override
{
1549 return target_
.env
->ReserveThreads(threads_to_be_reserved
, pri
);
1552 int ReleaseThreads(int threads_to_be_released
, Priority pri
) override
{
1553 return target_
.env
->ReleaseThreads(threads_to_be_released
, pri
);
1556 Status
GetTestDirectory(std::string
* path
) override
{
1557 return target_
.env
->GetTestDirectory(path
);
1559 Status
NewLogger(const std::string
& fname
,
1560 std::shared_ptr
<Logger
>* result
) override
{
1561 return target_
.env
->NewLogger(fname
, result
);
1563 uint64_t NowMicros() override
{ return target_
.env
->NowMicros(); }
1564 uint64_t NowNanos() override
{ return target_
.env
->NowNanos(); }
1565 uint64_t NowCPUNanos() override
{ return target_
.env
->NowCPUNanos(); }
1567 void SleepForMicroseconds(int micros
) override
{
1568 target_
.env
->SleepForMicroseconds(micros
);
1570 Status
GetHostName(char* name
, uint64_t len
) override
{
1571 return target_
.env
->GetHostName(name
, len
);
1573 Status
GetCurrentTime(int64_t* unix_time
) override
{
1574 return target_
.env
->GetCurrentTime(unix_time
);
1576 Status
GetAbsolutePath(const std::string
& db_path
,
1577 std::string
* output_path
) override
{
1578 return target_
.env
->GetAbsolutePath(db_path
, output_path
);
1580 void SetBackgroundThreads(int num
, Priority pri
) override
{
1581 return target_
.env
->SetBackgroundThreads(num
, pri
);
1583 int GetBackgroundThreads(Priority pri
) override
{
1584 return target_
.env
->GetBackgroundThreads(pri
);
1587 Status
SetAllowNonOwnerAccess(bool allow_non_owner_access
) override
{
1588 return target_
.env
->SetAllowNonOwnerAccess(allow_non_owner_access
);
1591 void IncBackgroundThreadsIfNeeded(int num
, Priority pri
) override
{
1592 return target_
.env
->IncBackgroundThreadsIfNeeded(num
, pri
);
1595 void LowerThreadPoolIOPriority(Priority pool
) override
{
1596 target_
.env
->LowerThreadPoolIOPriority(pool
);
1599 void LowerThreadPoolCPUPriority(Priority pool
) override
{
1600 target_
.env
->LowerThreadPoolCPUPriority(pool
);
1603 Status
LowerThreadPoolCPUPriority(Priority pool
, CpuPriority pri
) override
{
1604 return target_
.env
->LowerThreadPoolCPUPriority(pool
, pri
);
1607 std::string
TimeToString(uint64_t time
) override
{
1608 return target_
.env
->TimeToString(time
);
1611 Status
GetThreadList(std::vector
<ThreadStatus
>* thread_list
) override
{
1612 return target_
.env
->GetThreadList(thread_list
);
1615 ThreadStatusUpdater
* GetThreadStatusUpdater() const override
{
1616 return target_
.env
->GetThreadStatusUpdater();
1619 uint64_t GetThreadID() const override
{ return target_
.env
->GetThreadID(); }
1621 std::string
GenerateUniqueId() override
{
1622 return target_
.env
->GenerateUniqueId();
1625 EnvOptions
OptimizeForLogRead(const EnvOptions
& env_options
) const override
{
1626 return target_
.env
->OptimizeForLogRead(env_options
);
1628 EnvOptions
OptimizeForManifestRead(
1629 const EnvOptions
& env_options
) const override
{
1630 return target_
.env
->OptimizeForManifestRead(env_options
);
1632 EnvOptions
OptimizeForLogWrite(const EnvOptions
& env_options
,
1633 const DBOptions
& db_options
) const override
{
1634 return target_
.env
->OptimizeForLogWrite(env_options
, db_options
);
1636 EnvOptions
OptimizeForManifestWrite(
1637 const EnvOptions
& env_options
) const override
{
1638 return target_
.env
->OptimizeForManifestWrite(env_options
);
1640 EnvOptions
OptimizeForCompactionTableWrite(
1641 const EnvOptions
& env_options
,
1642 const ImmutableDBOptions
& immutable_ops
) const override
{
1643 return target_
.env
->OptimizeForCompactionTableWrite(env_options
,
1646 EnvOptions
OptimizeForCompactionTableRead(
1647 const EnvOptions
& env_options
,
1648 const ImmutableDBOptions
& db_options
) const override
{
1649 return target_
.env
->OptimizeForCompactionTableRead(env_options
, db_options
);
1651 EnvOptions
OptimizeForBlobFileRead(
1652 const EnvOptions
& env_options
,
1653 const ImmutableDBOptions
& db_options
) const override
{
1654 return target_
.env
->OptimizeForBlobFileRead(env_options
, db_options
);
1656 Status
GetFreeSpace(const std::string
& path
, uint64_t* diskfree
) override
{
1657 return target_
.env
->GetFreeSpace(path
, diskfree
);
1659 void SanitizeEnvOptions(EnvOptions
* env_opts
) const override
{
1660 target_
.env
->SanitizeEnvOptions(env_opts
);
1662 Status
PrepareOptions(const ConfigOptions
& options
) override
;
1663 #ifndef ROCKSDB_LITE
1664 std::string
SerializeOptions(const ConfigOptions
& config_options
,
1665 const std::string
& header
) const override
;
1666 #endif // ROCKSDB_LITE
1672 class SequentialFileWrapper
: public SequentialFile
{
1674 explicit SequentialFileWrapper(SequentialFile
* target
) : target_(target
) {}
1676 Status
Read(size_t n
, Slice
* result
, char* scratch
) override
{
1677 return target_
->Read(n
, result
, scratch
);
1679 Status
Skip(uint64_t n
) override
{ return target_
->Skip(n
); }
1680 bool use_direct_io() const override
{ return target_
->use_direct_io(); }
1681 size_t GetRequiredBufferAlignment() const override
{
1682 return target_
->GetRequiredBufferAlignment();
1684 Status
InvalidateCache(size_t offset
, size_t length
) override
{
1685 return target_
->InvalidateCache(offset
, length
);
1687 Status
PositionedRead(uint64_t offset
, size_t n
, Slice
* result
,
1688 char* scratch
) override
{
1689 return target_
->PositionedRead(offset
, n
, result
, scratch
);
1693 SequentialFile
* target_
;
1696 class RandomAccessFileWrapper
: public RandomAccessFile
{
1698 explicit RandomAccessFileWrapper(RandomAccessFile
* target
)
1699 : target_(target
) {}
1701 Status
Read(uint64_t offset
, size_t n
, Slice
* result
,
1702 char* scratch
) const override
{
1703 return target_
->Read(offset
, n
, result
, scratch
);
1705 Status
MultiRead(ReadRequest
* reqs
, size_t num_reqs
) override
{
1706 return target_
->MultiRead(reqs
, num_reqs
);
1708 Status
Prefetch(uint64_t offset
, size_t n
) override
{
1709 return target_
->Prefetch(offset
, n
);
1711 size_t GetUniqueId(char* id
, size_t max_size
) const override
{
1712 return target_
->GetUniqueId(id
, max_size
);
1714 void Hint(AccessPattern pattern
) override
{ target_
->Hint(pattern
); }
1715 bool use_direct_io() const override
{ return target_
->use_direct_io(); }
1716 size_t GetRequiredBufferAlignment() const override
{
1717 return target_
->GetRequiredBufferAlignment();
1719 Status
InvalidateCache(size_t offset
, size_t length
) override
{
1720 return target_
->InvalidateCache(offset
, length
);
1724 RandomAccessFile
* target_
;
1727 class WritableFileWrapper
: public WritableFile
{
1729 explicit WritableFileWrapper(WritableFile
* t
) : target_(t
) {}
1731 Status
Append(const Slice
& data
) override
{ return target_
->Append(data
); }
1732 Status
Append(const Slice
& data
,
1733 const DataVerificationInfo
& verification_info
) override
{
1734 return target_
->Append(data
, verification_info
);
1736 Status
PositionedAppend(const Slice
& data
, uint64_t offset
) override
{
1737 return target_
->PositionedAppend(data
, offset
);
1739 Status
PositionedAppend(
1740 const Slice
& data
, uint64_t offset
,
1741 const DataVerificationInfo
& verification_info
) override
{
1742 return target_
->PositionedAppend(data
, offset
, verification_info
);
1744 Status
Truncate(uint64_t size
) override
{ return target_
->Truncate(size
); }
1745 Status
Close() override
{ return target_
->Close(); }
1746 Status
Flush() override
{ return target_
->Flush(); }
1747 Status
Sync() override
{ return target_
->Sync(); }
1748 Status
Fsync() override
{ return target_
->Fsync(); }
1749 bool IsSyncThreadSafe() const override
{ return target_
->IsSyncThreadSafe(); }
1751 bool use_direct_io() const override
{ return target_
->use_direct_io(); }
1753 size_t GetRequiredBufferAlignment() const override
{
1754 return target_
->GetRequiredBufferAlignment();
1757 void SetIOPriority(Env::IOPriority pri
) override
{
1758 target_
->SetIOPriority(pri
);
1761 Env::IOPriority
GetIOPriority() override
{ return target_
->GetIOPriority(); }
1763 void SetWriteLifeTimeHint(Env::WriteLifeTimeHint hint
) override
{
1764 target_
->SetWriteLifeTimeHint(hint
);
1767 Env::WriteLifeTimeHint
GetWriteLifeTimeHint() override
{
1768 return target_
->GetWriteLifeTimeHint();
1771 uint64_t GetFileSize() override
{ return target_
->GetFileSize(); }
1773 void SetPreallocationBlockSize(size_t size
) override
{
1774 target_
->SetPreallocationBlockSize(size
);
1777 void GetPreallocationStatus(size_t* block_size
,
1778 size_t* last_allocated_block
) override
{
1779 target_
->GetPreallocationStatus(block_size
, last_allocated_block
);
1782 size_t GetUniqueId(char* id
, size_t max_size
) const override
{
1783 return target_
->GetUniqueId(id
, max_size
);
1786 Status
InvalidateCache(size_t offset
, size_t length
) override
{
1787 return target_
->InvalidateCache(offset
, length
);
1790 Status
RangeSync(uint64_t offset
, uint64_t nbytes
) override
{
1791 return target_
->RangeSync(offset
, nbytes
);
1794 void PrepareWrite(size_t offset
, size_t len
) override
{
1795 target_
->PrepareWrite(offset
, len
);
1798 Status
Allocate(uint64_t offset
, uint64_t len
) override
{
1799 return target_
->Allocate(offset
, len
);
1803 WritableFile
* target_
;
1806 class RandomRWFileWrapper
: public RandomRWFile
{
1808 explicit RandomRWFileWrapper(RandomRWFile
* target
) : target_(target
) {}
1810 bool use_direct_io() const override
{ return target_
->use_direct_io(); }
1811 size_t GetRequiredBufferAlignment() const override
{
1812 return target_
->GetRequiredBufferAlignment();
1814 Status
Write(uint64_t offset
, const Slice
& data
) override
{
1815 return target_
->Write(offset
, data
);
1817 Status
Read(uint64_t offset
, size_t n
, Slice
* result
,
1818 char* scratch
) const override
{
1819 return target_
->Read(offset
, n
, result
, scratch
);
1821 Status
Flush() override
{ return target_
->Flush(); }
1822 Status
Sync() override
{ return target_
->Sync(); }
1823 Status
Fsync() override
{ return target_
->Fsync(); }
1824 Status
Close() override
{ return target_
->Close(); }
1827 RandomRWFile
* target_
;
1830 class DirectoryWrapper
: public Directory
{
1832 explicit DirectoryWrapper(Directory
* target
) : target_(target
) {}
1834 Status
Fsync() override
{ return target_
->Fsync(); }
1835 Status
Close() override
{ return target_
->Close(); }
1836 size_t GetUniqueId(char* id
, size_t max_size
) const override
{
1837 return target_
->GetUniqueId(id
, max_size
);
1844 class LoggerWrapper
: public Logger
{
1846 explicit LoggerWrapper(Logger
* target
) : target_(target
) {}
1848 Status
Close() override
{ return target_
->Close(); }
1849 void LogHeader(const char* format
, va_list ap
) override
{
1850 return target_
->LogHeader(format
, ap
);
1852 void Logv(const char* format
, va_list ap
) override
{
1853 return target_
->Logv(format
, ap
);
1855 void Logv(const InfoLogLevel log_level
, const char* format
,
1856 va_list ap
) override
{
1857 return target_
->Logv(log_level
, format
, ap
);
1859 size_t GetLogFileSize() const override
{ return target_
->GetLogFileSize(); }
1860 void Flush() override
{ return target_
->Flush(); }
1861 InfoLogLevel
GetInfoLogLevel() const override
{
1862 return target_
->GetInfoLogLevel();
1864 void SetInfoLogLevel(const InfoLogLevel log_level
) override
{
1865 return target_
->SetInfoLogLevel(log_level
);
1872 // Returns a new environment that stores its data in memory and delegates
1873 // all non-file-storage tasks to base_env. The caller must delete the result
1874 // when it is no longer needed.
1875 // *base_env must remain live while the result is in use.
1876 Env
* NewMemEnv(Env
* base_env
);
1878 // Returns a new environment that measures function call times for filesystem
1879 // operations, reporting results to variables in PerfContext.
1880 // This is a factory method for TimedEnv defined in utilities/env_timed.cc.
1881 Env
* NewTimedEnv(Env
* base_env
);
1883 // Returns an instance of logger that can be used for storing informational
1885 // This is a factory method for EnvLogger declared in logging/env_logging.h
1886 Status
NewEnvLogger(const std::string
& fname
, Env
* env
,
1887 std::shared_ptr
<Logger
>* result
);
1889 // Creates a new Env based on Env::Default() but modified to use the specified
1891 std::unique_ptr
<Env
> NewCompositeEnv(const std::shared_ptr
<FileSystem
>& fs
);
1893 } // namespace ROCKSDB_NAMESPACE