1 // Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
2 // This source code is licensed under both the GPLv2 (found in the
3 // COPYING file in the root directory) and Apache 2.0 License
4 // (found in the LICENSE.Apache file in the root directory).
6 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
7 // Use of this source code is governed by a BSD-style license that can be
8 // found in the LICENSE file. See the AUTHORS file for names of contributors.
13 #ifndef __STDC_FORMAT_MACROS
14 #define __STDC_FORMAT_MACROS
23 #include "rocksdb/utilities/stackable_db.h"
25 #include "rocksdb/env.h"
26 #include "rocksdb/status.h"
30 struct BackupableDBOptions
{
31 // Where to keep the backup files. Has to be different than dbname_
32 // Best to set this to dbname_ + "/backups"
34 std::string backup_dir
;
36 // Backup Env object. It will be used for backup file I/O. If it's
37 // nullptr, backups will be written out using DBs Env. If it's
38 // non-nullptr, backup's I/O will be performed using this object.
39 // If you want to have backups on HDFS, use HDFS Env here!
43 // If share_table_files == true, backup will assume that table files with
44 // same name have the same contents. This enables incremental backups and
45 // avoids unnecessary data copies.
46 // If share_table_files == false, each backup will be on its own and will
47 // not share any data with other backups.
49 bool share_table_files
;
51 // Backup info and error messages will be written to info_log
56 // If sync == true, we can guarantee you'll get consistent backup even
57 // on a machine crash/reboot. Backup process is slower with sync enabled.
58 // If sync == false, we don't guarantee anything on machine reboot. However,
59 // chances are some of the backups are consistent.
63 // If true, it will delete whatever backups there are already
65 bool destroy_old_data
;
67 // If false, we won't backup log files. This option can be useful for backing
68 // up in-memory databases where log file are persisted, but table files are in
71 bool backup_log_files
;
73 // Max bytes that can be transferred in a second during backup.
74 // If 0, go as fast as you can
76 uint64_t backup_rate_limit
;
78 // Backup rate limiter. Used to control transfer speed for backup. If this is
79 // not null, backup_rate_limit is ignored.
81 std::shared_ptr
<RateLimiter
> backup_rate_limiter
{nullptr};
83 // Max bytes that can be transferred in a second during restore.
84 // If 0, go as fast as you can
86 uint64_t restore_rate_limit
;
88 // Restore rate limiter. Used to control transfer speed during restore. If
89 // this is not null, restore_rate_limit is ignored.
91 std::shared_ptr
<RateLimiter
> restore_rate_limiter
{nullptr};
93 // Only used if share_table_files is set to true. If true, will consider that
94 // backups can come from different databases, hence a sst is not uniquely
95 // identifed by its name, but by the triple (file name, crc32, file length)
97 // Note: this is an experimental option, and you'll need to set it manually
98 // *turn it on only if you know what you're doing*
99 bool share_files_with_checksum
;
101 // Up to this many background threads will copy files for CreateNewBackup()
102 // and RestoreDBFromBackup()
104 int max_background_operations
;
106 // During backup user can get callback every time next
107 // callback_trigger_interval_size bytes being copied.
109 uint64_t callback_trigger_interval_size
;
111 // When Open() is called, it will open at most this many of the latest
112 // non-corrupted backups.
114 // Note setting this to a non-default value prevents old files from being
115 // deleted in the shared directory, as we can't do proper ref-counting. If
116 // using this option, make sure to occasionally disable it (by resetting to
117 // INT_MAX) and run GarbageCollect to clean accumulated stale files.
120 int max_valid_backups_to_open
;
122 void Dump(Logger
* logger
) const;
124 explicit BackupableDBOptions(
125 const std::string
& _backup_dir
, Env
* _backup_env
= nullptr,
126 bool _share_table_files
= true, Logger
* _info_log
= nullptr,
127 bool _sync
= true, bool _destroy_old_data
= false,
128 bool _backup_log_files
= true, uint64_t _backup_rate_limit
= 0,
129 uint64_t _restore_rate_limit
= 0, int _max_background_operations
= 1,
130 uint64_t _callback_trigger_interval_size
= 4 * 1024 * 1024,
131 int _max_valid_backups_to_open
= INT_MAX
)
132 : backup_dir(_backup_dir
),
133 backup_env(_backup_env
),
134 share_table_files(_share_table_files
),
137 destroy_old_data(_destroy_old_data
),
138 backup_log_files(_backup_log_files
),
139 backup_rate_limit(_backup_rate_limit
),
140 restore_rate_limit(_restore_rate_limit
),
141 share_files_with_checksum(false),
142 max_background_operations(_max_background_operations
),
143 callback_trigger_interval_size(_callback_trigger_interval_size
),
144 max_valid_backups_to_open(_max_valid_backups_to_open
) {
145 assert(share_table_files
|| !share_files_with_checksum
);
149 struct RestoreOptions
{
150 // If true, restore won't overwrite the existing log files in wal_dir. It will
151 // also move all log files from archive directory to wal_dir. Use this option
152 // in combination with BackupableDBOptions::backup_log_files = false for
153 // persisting in-memory databases.
157 explicit RestoreOptions(bool _keep_log_files
= false)
158 : keep_log_files(_keep_log_files
) {}
161 typedef uint32_t BackupID
;
168 uint32_t number_files
;
169 std::string app_metadata
;
173 BackupInfo(BackupID _backup_id
, int64_t _timestamp
, uint64_t _size
,
174 uint32_t _number_files
, const std::string
& _app_metadata
)
175 : backup_id(_backup_id
),
176 timestamp(_timestamp
),
178 number_files(_number_files
),
179 app_metadata(_app_metadata
) {}
182 class BackupStatistics
{
185 number_success_backup
= 0;
186 number_fail_backup
= 0;
189 BackupStatistics(uint32_t _number_success_backup
,
190 uint32_t _number_fail_backup
)
191 : number_success_backup(_number_success_backup
),
192 number_fail_backup(_number_fail_backup
) {}
194 ~BackupStatistics() {}
196 void IncrementNumberSuccessBackup();
197 void IncrementNumberFailBackup();
199 uint32_t GetNumberSuccessBackup() const;
200 uint32_t GetNumberFailBackup() const;
202 std::string
ToString() const;
205 uint32_t number_success_backup
;
206 uint32_t number_fail_backup
;
209 // A backup engine for accessing information about backups and restoring from
211 class BackupEngineReadOnly
{
213 virtual ~BackupEngineReadOnly() {}
215 static Status
Open(Env
* db_env
, const BackupableDBOptions
& options
,
216 BackupEngineReadOnly
** backup_engine_ptr
);
218 // Returns info about backups in backup_info
219 // You can GetBackupInfo safely, even with other BackupEngine performing
220 // backups on the same directory
221 virtual void GetBackupInfo(std::vector
<BackupInfo
>* backup_info
) = 0;
223 // Returns info about corrupt backups in corrupt_backups
224 virtual void GetCorruptedBackups(
225 std::vector
<BackupID
>* corrupt_backup_ids
) = 0;
227 // Restoring DB from backup is NOT safe when there is another BackupEngine
228 // running that might call DeleteBackup() or PurgeOldBackups(). It is caller's
229 // responsibility to synchronize the operation, i.e. don't delete the backup
230 // when you're restoring from it
231 // See also the corresponding doc in BackupEngine
232 virtual Status
RestoreDBFromBackup(
233 BackupID backup_id
, const std::string
& db_dir
, const std::string
& wal_dir
,
234 const RestoreOptions
& restore_options
= RestoreOptions()) = 0;
236 // See the corresponding doc in BackupEngine
237 virtual Status
RestoreDBFromLatestBackup(
238 const std::string
& db_dir
, const std::string
& wal_dir
,
239 const RestoreOptions
& restore_options
= RestoreOptions()) = 0;
241 // checks that each file exists and that the size of the file matches our
242 // expectations. it does not check file checksum.
244 // If this BackupEngine created the backup, it compares the files' current
245 // sizes against the number of bytes written to them during creation.
246 // Otherwise, it compares the files' current sizes against their sizes when
247 // the BackupEngine was opened.
249 // Returns Status::OK() if all checks are good
250 virtual Status
VerifyBackup(BackupID backup_id
) = 0;
253 // A backup engine for creating new backups.
256 virtual ~BackupEngine() {}
258 // BackupableDBOptions have to be the same as the ones used in previous
259 // BackupEngines for the same backup directory.
260 static Status
Open(Env
* db_env
, const BackupableDBOptions
& options
,
261 BackupEngine
** backup_engine_ptr
);
263 // same as CreateNewBackup, but stores extra application metadata
264 // Flush will always trigger if 2PC is enabled.
265 // If write-ahead logs are disabled, set flush_before_backup=true to
266 // avoid losing unflushed key/value pairs from the memtable.
267 virtual Status
CreateNewBackupWithMetadata(
268 DB
* db
, const std::string
& app_metadata
, bool flush_before_backup
= false,
269 std::function
<void()> progress_callback
= []() {}) = 0;
271 // Captures the state of the database in the latest backup
272 // NOT a thread safe call
273 // Flush will always trigger if 2PC is enabled.
274 // If write-ahead logs are disabled, set flush_before_backup=true to
275 // avoid losing unflushed key/value pairs from the memtable.
276 virtual Status
CreateNewBackup(DB
* db
, bool flush_before_backup
= false,
277 std::function
<void()> progress_callback
=
279 return CreateNewBackupWithMetadata(db
, "", flush_before_backup
,
283 // deletes old backups, keeping latest num_backups_to_keep alive
284 virtual Status
PurgeOldBackups(uint32_t num_backups_to_keep
) = 0;
286 // deletes a specific backup
287 virtual Status
DeleteBackup(BackupID backup_id
) = 0;
289 // Call this from another thread if you want to stop the backup
290 // that is currently happening. It will return immediatelly, will
291 // not wait for the backup to stop.
292 // The backup will stop ASAP and the call to CreateNewBackup will
293 // return Status::Incomplete(). It will not clean up after itself, but
294 // the state will remain consistent. The state will be cleaned up
295 // next time you create BackupableDB or RestoreBackupableDB.
296 virtual void StopBackup() = 0;
298 // Returns info about backups in backup_info
299 virtual void GetBackupInfo(std::vector
<BackupInfo
>* backup_info
) = 0;
301 // Returns info about corrupt backups in corrupt_backups
302 virtual void GetCorruptedBackups(
303 std::vector
<BackupID
>* corrupt_backup_ids
) = 0;
305 // restore from backup with backup_id
306 // IMPORTANT -- if options_.share_table_files == true,
307 // options_.share_files_with_checksum == false, you restore DB from some
308 // backup that is not the latest, and you start creating new backups from the
309 // new DB, they will probably fail.
311 // Example: Let's say you have backups 1, 2, 3, 4, 5 and you restore 3.
312 // If you add new data to the DB and try creating a new backup now, the
313 // database will diverge from backups 4 and 5 and the new backup will fail.
314 // If you want to create new backup, you will first have to delete backups 4
316 virtual Status
RestoreDBFromBackup(
317 BackupID backup_id
, const std::string
& db_dir
, const std::string
& wal_dir
,
318 const RestoreOptions
& restore_options
= RestoreOptions()) = 0;
320 // restore from the latest backup
321 virtual Status
RestoreDBFromLatestBackup(
322 const std::string
& db_dir
, const std::string
& wal_dir
,
323 const RestoreOptions
& restore_options
= RestoreOptions()) = 0;
325 // checks that each file exists and that the size of the file matches our
326 // expectations. it does not check file checksum.
327 // Returns Status::OK() if all checks are good
328 virtual Status
VerifyBackup(BackupID backup_id
) = 0;
330 // Will delete all the files we don't need anymore
331 // It will do the full scan of the files/ directory and delete all the
332 // files that are not referenced.
333 virtual Status
GarbageCollect() = 0;
336 } // namespace rocksdb
337 #endif // ROCKSDB_LITE