1 // Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
2 // This source code is licensed under the BSD-style license found in the
3 // LICENSE file in the root directory of this source tree. An additional grant
4 // of patent rights can be found in the PATENTS file in the same directory.
6 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
7 // Use of this source code is governed by a BSD-style license that can be
8 // found in the LICENSE file. See the AUTHORS file for names of contributors.
13 #ifndef __STDC_FORMAT_MACROS
14 #define __STDC_FORMAT_MACROS
23 #include "rocksdb/utilities/stackable_db.h"
25 #include "rocksdb/env.h"
26 #include "rocksdb/status.h"
30 struct BackupableDBOptions
{
31 // Where to keep the backup files. Has to be different than dbname_
32 // Best to set this to dbname_ + "/backups"
34 std::string backup_dir
;
36 // Backup Env object. It will be used for backup file I/O. If it's
37 // nullptr, backups will be written out using DBs Env. If it's
38 // non-nullptr, backup's I/O will be performed using this object.
39 // If you want to have backups on HDFS, use HDFS Env here!
43 // If share_table_files == true, backup will assume that table files with
44 // same name have the same contents. This enables incremental backups and
45 // avoids unnecessary data copies.
46 // If share_table_files == false, each backup will be on its own and will
47 // not share any data with other backups.
49 bool share_table_files
;
51 // Backup info and error messages will be written to info_log
56 // If sync == true, we can guarantee you'll get consistent backup even
57 // on a machine crash/reboot. Backup process is slower with sync enabled.
58 // If sync == false, we don't guarantee anything on machine reboot. However,
59 // chances are some of the backups are consistent.
63 // If true, it will delete whatever backups there are already
65 bool destroy_old_data
;
67 // If false, we won't backup log files. This option can be useful for backing
68 // up in-memory databases where log file are persisted, but table files are in
71 bool backup_log_files
;
73 // Max bytes that can be transferred in a second during backup.
74 // If 0, go as fast as you can
76 uint64_t backup_rate_limit
;
78 // Backup rate limiter. Used to control transfer speed for backup. If this is
79 // not null, backup_rate_limit is ignored.
81 std::shared_ptr
<RateLimiter
> backup_rate_limiter
{nullptr};
83 // Max bytes that can be transferred in a second during restore.
84 // If 0, go as fast as you can
86 uint64_t restore_rate_limit
;
88 // Restore rate limiter. Used to control transfer speed during restore. If
89 // this is not null, restore_rate_limit is ignored.
91 std::shared_ptr
<RateLimiter
> restore_rate_limiter
{nullptr};
93 // Only used if share_table_files is set to true. If true, will consider that
94 // backups can come from different databases, hence a sst is not uniquely
95 // identifed by its name, but by the triple (file name, crc32, file length)
97 // Note: this is an experimental option, and you'll need to set it manually
98 // *turn it on only if you know what you're doing*
99 bool share_files_with_checksum
;
101 // Up to this many background threads will copy files for CreateNewBackup()
102 // and RestoreDBFromBackup()
104 int max_background_operations
;
106 // During backup user can get callback every time next
107 // callback_trigger_interval_size bytes being copied.
109 uint64_t callback_trigger_interval_size
;
111 // When Open() is called, it will open at most this many of the latest
112 // non-corrupted backups. If 0, it will open all available backups.
114 int max_valid_backups_to_open
;
116 void Dump(Logger
* logger
) const;
118 explicit BackupableDBOptions(
119 const std::string
& _backup_dir
, Env
* _backup_env
= nullptr,
120 bool _share_table_files
= true, Logger
* _info_log
= nullptr,
121 bool _sync
= true, bool _destroy_old_data
= false,
122 bool _backup_log_files
= true, uint64_t _backup_rate_limit
= 0,
123 uint64_t _restore_rate_limit
= 0, int _max_background_operations
= 1,
124 uint64_t _callback_trigger_interval_size
= 4 * 1024 * 1024,
125 int _max_valid_backups_to_open
= 0)
126 : backup_dir(_backup_dir
),
127 backup_env(_backup_env
),
128 share_table_files(_share_table_files
),
131 destroy_old_data(_destroy_old_data
),
132 backup_log_files(_backup_log_files
),
133 backup_rate_limit(_backup_rate_limit
),
134 restore_rate_limit(_restore_rate_limit
),
135 share_files_with_checksum(false),
136 max_background_operations(_max_background_operations
),
137 callback_trigger_interval_size(_callback_trigger_interval_size
),
138 max_valid_backups_to_open(_max_valid_backups_to_open
) {
139 assert(share_table_files
|| !share_files_with_checksum
);
143 struct RestoreOptions
{
144 // If true, restore won't overwrite the existing log files in wal_dir. It will
145 // also move all log files from archive directory to wal_dir. Use this option
146 // in combination with BackupableDBOptions::backup_log_files = false for
147 // persisting in-memory databases.
151 explicit RestoreOptions(bool _keep_log_files
= false)
152 : keep_log_files(_keep_log_files
) {}
155 typedef uint32_t BackupID
;
162 uint32_t number_files
;
163 std::string app_metadata
;
167 BackupInfo(BackupID _backup_id
, int64_t _timestamp
, uint64_t _size
,
168 uint32_t _number_files
, const std::string
& _app_metadata
)
169 : backup_id(_backup_id
),
170 timestamp(_timestamp
),
172 number_files(_number_files
),
173 app_metadata(_app_metadata
) {}
176 class BackupStatistics
{
179 number_success_backup
= 0;
180 number_fail_backup
= 0;
183 BackupStatistics(uint32_t _number_success_backup
,
184 uint32_t _number_fail_backup
)
185 : number_success_backup(_number_success_backup
),
186 number_fail_backup(_number_fail_backup
) {}
188 ~BackupStatistics() {}
190 void IncrementNumberSuccessBackup();
191 void IncrementNumberFailBackup();
193 uint32_t GetNumberSuccessBackup() const;
194 uint32_t GetNumberFailBackup() const;
196 std::string
ToString() const;
199 uint32_t number_success_backup
;
200 uint32_t number_fail_backup
;
203 // A backup engine for accessing information about backups and restoring from
205 class BackupEngineReadOnly
{
207 virtual ~BackupEngineReadOnly() {}
209 static Status
Open(Env
* db_env
, const BackupableDBOptions
& options
,
210 BackupEngineReadOnly
** backup_engine_ptr
);
212 // Returns info about backups in backup_info
213 // You can GetBackupInfo safely, even with other BackupEngine performing
214 // backups on the same directory
215 virtual void GetBackupInfo(std::vector
<BackupInfo
>* backup_info
) = 0;
217 // Returns info about corrupt backups in corrupt_backups
218 virtual void GetCorruptedBackups(
219 std::vector
<BackupID
>* corrupt_backup_ids
) = 0;
221 // Restoring DB from backup is NOT safe when there is another BackupEngine
222 // running that might call DeleteBackup() or PurgeOldBackups(). It is caller's
223 // responsibility to synchronize the operation, i.e. don't delete the backup
224 // when you're restoring from it
225 // See also the corresponding doc in BackupEngine
226 virtual Status
RestoreDBFromBackup(
227 BackupID backup_id
, const std::string
& db_dir
, const std::string
& wal_dir
,
228 const RestoreOptions
& restore_options
= RestoreOptions()) = 0;
230 // See the corresponding doc in BackupEngine
231 virtual Status
RestoreDBFromLatestBackup(
232 const std::string
& db_dir
, const std::string
& wal_dir
,
233 const RestoreOptions
& restore_options
= RestoreOptions()) = 0;
235 // checks that each file exists and that the size of the file matches our
236 // expectations. it does not check file checksum.
238 // If this BackupEngine created the backup, it compares the files' current
239 // sizes against the number of bytes written to them during creation.
240 // Otherwise, it compares the files' current sizes against their sizes when
241 // the BackupEngine was opened.
243 // Returns Status::OK() if all checks are good
244 virtual Status
VerifyBackup(BackupID backup_id
) = 0;
247 // A backup engine for creating new backups.
250 virtual ~BackupEngine() {}
252 // BackupableDBOptions have to be the same as the ones used in previous
253 // BackupEngines for the same backup directory.
254 static Status
Open(Env
* db_env
,
255 const BackupableDBOptions
& options
,
256 BackupEngine
** backup_engine_ptr
);
258 // same as CreateNewBackup, but stores extra application metadata
259 // Flush will always trigger if 2PC is enabled.
260 virtual Status
CreateNewBackupWithMetadata(
261 DB
* db
, const std::string
& app_metadata
, bool flush_before_backup
= false,
262 std::function
<void()> progress_callback
= []() {}) = 0;
264 // Captures the state of the database in the latest backup
265 // NOT a thread safe call
266 // Flush will always trigger if 2PC is enabled.
267 virtual Status
CreateNewBackup(DB
* db
, bool flush_before_backup
= false,
268 std::function
<void()> progress_callback
=
270 return CreateNewBackupWithMetadata(db
, "", flush_before_backup
,
274 // deletes old backups, keeping latest num_backups_to_keep alive
275 virtual Status
PurgeOldBackups(uint32_t num_backups_to_keep
) = 0;
277 // deletes a specific backup
278 virtual Status
DeleteBackup(BackupID backup_id
) = 0;
280 // Call this from another thread if you want to stop the backup
281 // that is currently happening. It will return immediatelly, will
282 // not wait for the backup to stop.
283 // The backup will stop ASAP and the call to CreateNewBackup will
284 // return Status::Incomplete(). It will not clean up after itself, but
285 // the state will remain consistent. The state will be cleaned up
286 // next time you create BackupableDB or RestoreBackupableDB.
287 virtual void StopBackup() = 0;
289 // Returns info about backups in backup_info
290 virtual void GetBackupInfo(std::vector
<BackupInfo
>* backup_info
) = 0;
292 // Returns info about corrupt backups in corrupt_backups
293 virtual void GetCorruptedBackups(
294 std::vector
<BackupID
>* corrupt_backup_ids
) = 0;
296 // restore from backup with backup_id
297 // IMPORTANT -- if options_.share_table_files == true,
298 // options_.share_files_with_checksum == false, you restore DB from some
299 // backup that is not the latest, and you start creating new backups from the
300 // new DB, they will probably fail.
302 // Example: Let's say you have backups 1, 2, 3, 4, 5 and you restore 3.
303 // If you add new data to the DB and try creating a new backup now, the
304 // database will diverge from backups 4 and 5 and the new backup will fail.
305 // If you want to create new backup, you will first have to delete backups 4
307 virtual Status
RestoreDBFromBackup(
308 BackupID backup_id
, const std::string
& db_dir
, const std::string
& wal_dir
,
309 const RestoreOptions
& restore_options
= RestoreOptions()) = 0;
311 // restore from the latest backup
312 virtual Status
RestoreDBFromLatestBackup(
313 const std::string
& db_dir
, const std::string
& wal_dir
,
314 const RestoreOptions
& restore_options
= RestoreOptions()) = 0;
316 // checks that each file exists and that the size of the file matches our
317 // expectations. it does not check file checksum.
318 // Returns Status::OK() if all checks are good
319 virtual Status
VerifyBackup(BackupID backup_id
) = 0;
321 // Will delete all the files we don't need anymore
322 // It will do the full scan of the files/ directory and delete all the
323 // files that are not referenced.
324 virtual Status
GarbageCollect() = 0;
327 } // namespace rocksdb
328 #endif // ROCKSDB_LITE