]>
Commit | Line | Data |
---|---|---|
c96a4a38 DM |
1 | From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 |
2 | From: Stefan Reiter <s.reiter@proxmox.com> | |
3 | Date: Mon, 29 Jun 2020 11:06:03 +0200 | |
4 | Subject: [PATCH] PVE-Backup: Add dirty-bitmap tracking for incremental backups | |
5 | ||
6 | Uses QEMU's existing MIRROR_SYNC_MODE_BITMAP and a dirty-bitmap on top | |
7 | of all backed-up drives. This will only execute the data-write callback | |
8 | for any changed chunks, the PBS rust code will reuse chunks from the | |
9 | previous index for everything it doesn't receive if reuse_index is true. | |
10 | ||
11 | On error or cancellation, remove all dirty bitmaps to ensure | |
12 | consistency. | |
13 | ||
20be7fa0 TL |
14 | Add PBS/incremental specific information to query backup info QMP and |
15 | HMP commands. | |
16 | ||
c96a4a38 DM |
17 | Only supported for PBS backups. |
18 | ||
19 | Signed-off-by: Stefan Reiter <s.reiter@proxmox.com> | |
20 | Signed-off-by: Dietmar Maurer <dietmar@proxmox.com> | |
20be7fa0 | 21 | Signed-off-by: Thomas Lamprecht <t.lamprecht@proxmox.com> |
c96a4a38 DM |
22 | --- |
23 | block/monitor/block-hmp-cmds.c | 1 + | |
20be7fa0 | 24 | monitor/hmp-cmds.c | 45 ++++++++++++---- |
c96a4a38 DM |
25 | proxmox-backup-client.c | 3 +- |
26 | proxmox-backup-client.h | 1 + | |
20be7fa0 TL |
27 | pve-backup.c | 95 ++++++++++++++++++++++++++++++---- |
28 | qapi/block-core.json | 12 ++++- | |
29 | 6 files changed, 134 insertions(+), 23 deletions(-) | |
c96a4a38 DM |
30 | |
31 | diff --git a/block/monitor/block-hmp-cmds.c b/block/monitor/block-hmp-cmds.c | |
817b7667 | 32 | index 9ba7c774a2..056d14deee 100644 |
c96a4a38 DM |
33 | --- a/block/monitor/block-hmp-cmds.c |
34 | +++ b/block/monitor/block-hmp-cmds.c | |
817b7667 | 35 | @@ -1039,6 +1039,7 @@ void hmp_backup(Monitor *mon, const QDict *qdict) |
c96a4a38 DM |
36 | false, NULL, // PBS fingerprint |
37 | false, NULL, // PBS backup-id | |
38 | false, 0, // PBS backup-time | |
39 | + false, false, // PBS incremental | |
40 | true, dir ? BACKUP_FORMAT_DIR : BACKUP_FORMAT_VMA, | |
41 | false, NULL, false, NULL, !!devlist, | |
42 | devlist, qdict_haskey(qdict, "speed"), speed, &error); | |
20be7fa0 | 43 | diff --git a/monitor/hmp-cmds.c b/monitor/hmp-cmds.c |
817b7667 | 44 | index 182e79c943..604026bb37 100644 |
20be7fa0 TL |
45 | --- a/monitor/hmp-cmds.c |
46 | +++ b/monitor/hmp-cmds.c | |
817b7667 | 47 | @@ -221,19 +221,42 @@ void hmp_info_backup(Monitor *mon, const QDict *qdict) |
20be7fa0 TL |
48 | monitor_printf(mon, "End time: %s", ctime(&info->end_time)); |
49 | } | |
50 | ||
51 | - int per = (info->has_total && info->total && | |
52 | - info->has_transferred && info->transferred) ? | |
53 | - (info->transferred * 100)/info->total : 0; | |
54 | - int zero_per = (info->has_total && info->total && | |
55 | - info->has_zero_bytes && info->zero_bytes) ? | |
56 | - (info->zero_bytes * 100)/info->total : 0; | |
57 | monitor_printf(mon, "Backup file: %s\n", info->backup_file); | |
58 | monitor_printf(mon, "Backup uuid: %s\n", info->uuid); | |
59 | - monitor_printf(mon, "Total size: %zd\n", info->total); | |
60 | - monitor_printf(mon, "Transferred bytes: %zd (%d%%)\n", | |
61 | - info->transferred, per); | |
62 | - monitor_printf(mon, "Zero bytes: %zd (%d%%)\n", | |
63 | - info->zero_bytes, zero_per); | |
64 | + | |
65 | + if (!(info->has_total && info->total)) { | |
66 | + // this should not happen normally | |
67 | + monitor_printf(mon, "Total size: %d\n", 0); | |
68 | + } else { | |
69 | + bool incremental = false; | |
70 | + size_t total_or_dirty = info->total; | |
71 | + if (info->has_transferred) { | |
72 | + if (info->has_dirty && info->dirty) { | |
73 | + if (info->dirty < info->total) { | |
74 | + total_or_dirty = info->dirty; | |
75 | + incremental = true; | |
76 | + } | |
77 | + } | |
78 | + } | |
79 | + | |
80 | + int per = (info->transferred * 100)/total_or_dirty; | |
81 | + | |
82 | + monitor_printf(mon, "Backup mode: %s\n", incremental ? "incremental" : "full"); | |
83 | + | |
84 | + int zero_per = (info->has_zero_bytes && info->zero_bytes) ? | |
85 | + (info->zero_bytes * 100)/info->total : 0; | |
86 | + monitor_printf(mon, "Total size: %zd\n", info->total); | |
87 | + monitor_printf(mon, "Transferred bytes: %zd (%d%%)\n", | |
88 | + info->transferred, per); | |
89 | + monitor_printf(mon, "Zero bytes: %zd (%d%%)\n", | |
90 | + info->zero_bytes, zero_per); | |
91 | + | |
92 | + if (info->has_reused) { | |
93 | + int reused_per = (info->reused * 100)/total_or_dirty; | |
94 | + monitor_printf(mon, "Reused bytes: %zd (%d%%)\n", | |
95 | + info->reused, reused_per); | |
96 | + } | |
97 | + } | |
98 | } | |
99 | ||
100 | qapi_free_BackupStatus(info); | |
c96a4a38 | 101 | diff --git a/proxmox-backup-client.c b/proxmox-backup-client.c |
72ae34ec | 102 | index a8f6653a81..4ce7bc0b5e 100644 |
c96a4a38 DM |
103 | --- a/proxmox-backup-client.c |
104 | +++ b/proxmox-backup-client.c | |
72ae34ec | 105 | @@ -89,6 +89,7 @@ proxmox_backup_co_register_image( |
c96a4a38 DM |
106 | ProxmoxBackupHandle *pbs, |
107 | const char *device_name, | |
108 | uint64_t size, | |
109 | + bool incremental, | |
110 | Error **errp) | |
111 | { | |
112 | Coroutine *co = qemu_coroutine_self(); | |
72ae34ec | 113 | @@ -98,7 +99,7 @@ proxmox_backup_co_register_image( |
c96a4a38 DM |
114 | int pbs_res = -1; |
115 | ||
116 | proxmox_backup_register_image_async( | |
117 | - pbs, device_name, size ,proxmox_backup_schedule_wake, &waker, &pbs_res, &pbs_err); | |
118 | + pbs, device_name, size, incremental, proxmox_backup_schedule_wake, &waker, &pbs_res, &pbs_err); | |
119 | qemu_coroutine_yield(); | |
120 | if (pbs_res < 0) { | |
121 | if (errp) error_setg(errp, "backup register image failed: %s", pbs_err ? pbs_err : "unknown error"); | |
122 | diff --git a/proxmox-backup-client.h b/proxmox-backup-client.h | |
72ae34ec | 123 | index 1dda8b7d8f..8cbf645b2c 100644 |
c96a4a38 DM |
124 | --- a/proxmox-backup-client.h |
125 | +++ b/proxmox-backup-client.h | |
72ae34ec | 126 | @@ -32,6 +32,7 @@ proxmox_backup_co_register_image( |
c96a4a38 DM |
127 | ProxmoxBackupHandle *pbs, |
128 | const char *device_name, | |
129 | uint64_t size, | |
130 | + bool incremental, | |
131 | Error **errp); | |
132 | ||
133 | ||
134 | diff --git a/pve-backup.c b/pve-backup.c | |
d333327a | 135 | index d40f3f2fd6..d50f03a050 100644 |
c96a4a38 DM |
136 | --- a/pve-backup.c |
137 | +++ b/pve-backup.c | |
138 | @@ -28,6 +28,8 @@ | |
139 | * | |
140 | */ | |
141 | ||
142 | +const char *PBS_BITMAP_NAME = "pbs-incremental-dirty-bitmap"; | |
143 | + | |
144 | static struct PVEBackupState { | |
145 | struct { | |
146 | // Everithing accessed from qmp_backup_query command is protected using lock | |
20be7fa0 TL |
147 | @@ -39,7 +41,9 @@ static struct PVEBackupState { |
148 | uuid_t uuid; | |
149 | char uuid_str[37]; | |
150 | size_t total; | |
151 | + size_t dirty; | |
152 | size_t transferred; | |
153 | + size_t reused; | |
154 | size_t zero_bytes; | |
155 | } stat; | |
156 | int64_t speed; | |
157 | @@ -66,6 +70,7 @@ typedef struct PVEBackupDevInfo { | |
c96a4a38 DM |
158 | uint8_t dev_id; |
159 | bool completed; | |
160 | char targetfile[PATH_MAX]; | |
161 | + BdrvDirtyBitmap *bitmap; | |
162 | BlockDriverState *target; | |
163 | } PVEBackupDevInfo; | |
164 | ||
20be7fa0 TL |
165 | @@ -105,11 +110,12 @@ static bool pvebackup_error_or_canceled(void) |
166 | return error_or_canceled; | |
167 | } | |
168 | ||
169 | -static void pvebackup_add_transfered_bytes(size_t transferred, size_t zero_bytes) | |
170 | +static void pvebackup_add_transfered_bytes(size_t transferred, size_t zero_bytes, size_t reused) | |
171 | { | |
172 | qemu_mutex_lock(&backup_state.stat.lock); | |
173 | backup_state.stat.zero_bytes += zero_bytes; | |
174 | backup_state.stat.transferred += transferred; | |
175 | + backup_state.stat.reused += reused; | |
176 | qemu_mutex_unlock(&backup_state.stat.lock); | |
177 | } | |
178 | ||
179 | @@ -148,7 +154,8 @@ pvebackup_co_dump_pbs_cb( | |
180 | pvebackup_propagate_error(local_err); | |
181 | return pbs_res; | |
182 | } else { | |
183 | - pvebackup_add_transfered_bytes(size, !buf ? size : 0); | |
184 | + size_t reused = (pbs_res == 0) ? size : 0; | |
185 | + pvebackup_add_transfered_bytes(size, !buf ? size : 0, reused); | |
186 | } | |
187 | ||
188 | return size; | |
189 | @@ -208,11 +215,11 @@ pvebackup_co_dump_vma_cb( | |
190 | } else { | |
191 | if (remaining >= VMA_CLUSTER_SIZE) { | |
192 | assert(ret == VMA_CLUSTER_SIZE); | |
193 | - pvebackup_add_transfered_bytes(VMA_CLUSTER_SIZE, zero_bytes); | |
194 | + pvebackup_add_transfered_bytes(VMA_CLUSTER_SIZE, zero_bytes, 0); | |
195 | remaining -= VMA_CLUSTER_SIZE; | |
196 | } else { | |
197 | assert(ret == remaining); | |
198 | - pvebackup_add_transfered_bytes(remaining, zero_bytes); | |
199 | + pvebackup_add_transfered_bytes(remaining, zero_bytes, 0); | |
200 | remaining = 0; | |
201 | } | |
202 | } | |
203 | @@ -248,6 +255,18 @@ static void coroutine_fn pvebackup_co_cleanup(void *unused) | |
c96a4a38 DM |
204 | if (local_err != NULL) { |
205 | pvebackup_propagate_error(local_err); | |
206 | } | |
207 | + } else { | |
208 | + // on error or cancel we cannot ensure synchronization of dirty | |
209 | + // bitmaps with backup server, so remove all and do full backup next | |
210 | + GList *l = backup_state.di_list; | |
211 | + while (l) { | |
212 | + PVEBackupDevInfo *di = (PVEBackupDevInfo *)l->data; | |
213 | + l = g_list_next(l); | |
214 | + | |
215 | + if (di->bitmap) { | |
216 | + bdrv_release_dirty_bitmap(di->bitmap); | |
217 | + } | |
218 | + } | |
219 | } | |
220 | ||
221 | proxmox_backup_disconnect(backup_state.pbs); | |
20be7fa0 TL |
222 | @@ -303,6 +322,12 @@ static void pvebackup_complete_cb(void *opaque, int ret) |
223 | // remove self from job queue | |
224 | backup_state.di_list = g_list_remove(backup_state.di_list, di); | |
225 | ||
226 | + if (di->bitmap && ret < 0) { | |
227 | + // on error or cancel we cannot ensure synchronization of dirty | |
228 | + // bitmaps with backup server, so remove all and do full backup next | |
229 | + bdrv_release_dirty_bitmap(di->bitmap); | |
230 | + } | |
231 | + | |
232 | g_free(di); | |
233 | ||
234 | qemu_mutex_unlock(&backup_state.backup_mutex); | |
235 | @@ -470,12 +495,18 @@ static bool create_backup_jobs(void) { | |
c96a4a38 DM |
236 | |
237 | assert(di->target != NULL); | |
238 | ||
239 | + MirrorSyncMode sync_mode = MIRROR_SYNC_MODE_FULL; | |
240 | + BitmapSyncMode bitmap_mode = BITMAP_SYNC_MODE_NEVER; | |
241 | + if (di->bitmap) { | |
242 | + sync_mode = MIRROR_SYNC_MODE_BITMAP; | |
243 | + bitmap_mode = BITMAP_SYNC_MODE_ON_SUCCESS; | |
244 | + } | |
245 | AioContext *aio_context = bdrv_get_aio_context(di->bs); | |
246 | aio_context_acquire(aio_context); | |
247 | ||
248 | BlockJob *job = backup_job_create( | |
249 | - NULL, di->bs, di->target, backup_state.speed, MIRROR_SYNC_MODE_FULL, NULL, | |
250 | - BITMAP_SYNC_MODE_NEVER, false, NULL, BLOCKDEV_ON_ERROR_REPORT, BLOCKDEV_ON_ERROR_REPORT, | |
251 | + NULL, di->bs, di->target, backup_state.speed, sync_mode, di->bitmap, | |
252 | + bitmap_mode, false, NULL, BLOCKDEV_ON_ERROR_REPORT, BLOCKDEV_ON_ERROR_REPORT, | |
253 | JOB_DEFAULT, pvebackup_complete_cb, di, 1, NULL, &local_err); | |
254 | ||
255 | aio_context_release(aio_context); | |
20be7fa0 | 256 | @@ -526,6 +557,8 @@ typedef struct QmpBackupTask { |
c96a4a38 DM |
257 | const char *fingerprint; |
258 | bool has_fingerprint; | |
259 | int64_t backup_time; | |
260 | + bool has_incremental; | |
261 | + bool incremental; | |
262 | bool has_format; | |
263 | BackupFormat format; | |
264 | bool has_config_file; | |
d333327a | 265 | @@ -617,6 +650,7 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque) |
20be7fa0 TL |
266 | } |
267 | ||
268 | size_t total = 0; | |
269 | + size_t dirty = 0; | |
270 | ||
271 | l = di_list; | |
272 | while (l) { | |
d333327a | 273 | @@ -654,6 +688,8 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque) |
c96a4a38 DM |
274 | int dump_cb_block_size = PROXMOX_BACKUP_DEFAULT_CHUNK_SIZE; // Hardcoded (4M) |
275 | firewall_name = "fw.conf"; | |
276 | ||
277 | + bool incremental = task->has_incremental && task->incremental; | |
278 | + | |
279 | char *pbs_err = NULL; | |
280 | pbs = proxmox_backup_new( | |
281 | task->backup_file, | |
d333327a | 282 | @@ -673,7 +709,8 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque) |
c96a4a38 DM |
283 | goto err; |
284 | } | |
285 | ||
286 | - if (proxmox_backup_co_connect(pbs, task->errp) < 0) | |
287 | + int connect_result = proxmox_backup_co_connect(pbs, task->errp); | |
288 | + if (connect_result < 0) | |
289 | goto err; | |
290 | ||
291 | /* register all devices */ | |
d333327a | 292 | @@ -684,9 +721,32 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque) |
c96a4a38 DM |
293 | |
294 | const char *devname = bdrv_get_device_name(di->bs); | |
295 | ||
296 | - int dev_id = proxmox_backup_co_register_image(pbs, devname, di->size, task->errp); | |
297 | - if (dev_id < 0) | |
298 | + BdrvDirtyBitmap *bitmap = bdrv_find_dirty_bitmap(di->bs, PBS_BITMAP_NAME); | |
299 | + | |
300 | + bool use_incremental = false; | |
301 | + if (incremental) { | |
302 | + if (bitmap == NULL) { | |
303 | + bitmap = bdrv_create_dirty_bitmap(di->bs, dump_cb_block_size, PBS_BITMAP_NAME, task->errp); | |
304 | + if (!bitmap) { | |
305 | + goto err; | |
306 | + } | |
307 | + /* mark entire bitmap as dirty to make full backup first */ | |
308 | + bdrv_set_dirty_bitmap(bitmap, 0, di->size); | |
20be7fa0 | 309 | + dirty += di->size; |
c96a4a38 DM |
310 | + } else { |
311 | + use_incremental = true; | |
20be7fa0 | 312 | + dirty += bdrv_get_dirty_count(bitmap); |
c96a4a38 DM |
313 | + } |
314 | + di->bitmap = bitmap; | |
315 | + } else if (bitmap != NULL) { | |
20be7fa0 | 316 | + dirty += di->size; |
c96a4a38 DM |
317 | + bdrv_release_dirty_bitmap(bitmap); |
318 | + } | |
319 | + | |
320 | + int dev_id = proxmox_backup_co_register_image(pbs, devname, di->size, use_incremental, task->errp); | |
321 | + if (dev_id < 0) { | |
322 | goto err; | |
323 | + } | |
324 | ||
325 | if (!(di->target = bdrv_backup_dump_create(dump_cb_block_size, di->size, pvebackup_co_dump_pbs_cb, di, task->errp))) { | |
326 | goto err; | |
d333327a | 327 | @@ -695,6 +755,8 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque) |
20be7fa0 TL |
328 | di->dev_id = dev_id; |
329 | } | |
330 | } else if (format == BACKUP_FORMAT_VMA) { | |
331 | + dirty = total; | |
332 | + | |
333 | vmaw = vma_writer_create(task->backup_file, uuid, &local_err); | |
334 | if (!vmaw) { | |
335 | if (local_err) { | |
d333327a | 336 | @@ -722,6 +784,8 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque) |
20be7fa0 TL |
337 | } |
338 | } | |
339 | } else if (format == BACKUP_FORMAT_DIR) { | |
340 | + dirty = total; | |
341 | + | |
342 | if (mkdir(task->backup_file, 0640) != 0) { | |
343 | error_setg_errno(task->errp, errno, "can't create directory '%s'\n", | |
344 | task->backup_file); | |
d333327a | 345 | @@ -794,8 +858,10 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque) |
20be7fa0 TL |
346 | char *uuid_str = g_strdup(backup_state.stat.uuid_str); |
347 | ||
348 | backup_state.stat.total = total; | |
349 | + backup_state.stat.dirty = dirty; | |
350 | backup_state.stat.transferred = 0; | |
351 | backup_state.stat.zero_bytes = 0; | |
15b9c76e | 352 | + backup_state.stat.reused = dirty >= total ? 0 : total - dirty; |
20be7fa0 TL |
353 | |
354 | qemu_mutex_unlock(&backup_state.stat.lock); | |
355 | ||
d333327a | 356 | @@ -819,6 +885,10 @@ err: |
c96a4a38 DM |
357 | PVEBackupDevInfo *di = (PVEBackupDevInfo *)l->data; |
358 | l = g_list_next(l); | |
359 | ||
360 | + if (di->bitmap) { | |
361 | + bdrv_release_dirty_bitmap(di->bitmap); | |
362 | + } | |
363 | + | |
364 | if (di->target) { | |
365 | bdrv_unref(di->target); | |
366 | } | |
d333327a | 367 | @@ -860,6 +930,7 @@ UuidInfo *qmp_backup( |
c96a4a38 DM |
368 | bool has_fingerprint, const char *fingerprint, |
369 | bool has_backup_id, const char *backup_id, | |
370 | bool has_backup_time, int64_t backup_time, | |
371 | + bool has_incremental, bool incremental, | |
372 | bool has_format, BackupFormat format, | |
373 | bool has_config_file, const char *config_file, | |
374 | bool has_firewall_file, const char *firewall_file, | |
d333327a | 375 | @@ -878,6 +949,8 @@ UuidInfo *qmp_backup( |
c96a4a38 DM |
376 | .backup_id = backup_id, |
377 | .has_backup_time = has_backup_time, | |
378 | .backup_time = backup_time, | |
379 | + .has_incremental = has_incremental, | |
380 | + .incremental = incremental, | |
381 | .has_format = has_format, | |
382 | .format = format, | |
383 | .has_config_file = has_config_file, | |
d333327a | 384 | @@ -946,10 +1019,14 @@ BackupStatus *qmp_query_backup(Error **errp) |
20be7fa0 TL |
385 | |
386 | info->has_total = true; | |
387 | info->total = backup_state.stat.total; | |
388 | + info->has_dirty = true; | |
389 | + info->dirty = backup_state.stat.dirty; | |
390 | info->has_zero_bytes = true; | |
391 | info->zero_bytes = backup_state.stat.zero_bytes; | |
392 | info->has_transferred = true; | |
393 | info->transferred = backup_state.stat.transferred; | |
394 | + info->has_reused = true; | |
395 | + info->reused = backup_state.stat.reused; | |
396 | ||
397 | qemu_mutex_unlock(&backup_state.stat.lock); | |
398 | ||
c96a4a38 | 399 | diff --git a/qapi/block-core.json b/qapi/block-core.json |
817b7667 | 400 | index 9054db608c..aadd5329b3 100644 |
c96a4a38 DM |
401 | --- a/qapi/block-core.json |
402 | +++ b/qapi/block-core.json | |
817b7667 | 403 | @@ -758,8 +758,13 @@ |
20be7fa0 TL |
404 | # |
405 | # @total: total amount of bytes involved in the backup process | |
406 | # | |
407 | +# @dirty: with incremental mode, this is the amount of bytes involved | |
408 | +# in the backup process which are marked dirty. | |
409 | +# | |
410 | # @transferred: amount of bytes already backed up. | |
411 | # | |
412 | +# @reused: amount of bytes reused due to deduplication. | |
413 | +# | |
414 | # @zero-bytes: amount of 'zero' bytes detected. | |
415 | # | |
416 | # @start-time: time (epoch) when backup job started. | |
817b7667 | 417 | @@ -772,8 +777,8 @@ |
20be7fa0 TL |
418 | # |
419 | ## | |
420 | { 'struct': 'BackupStatus', | |
421 | - 'data': {'*status': 'str', '*errmsg': 'str', '*total': 'int', | |
422 | - '*transferred': 'int', '*zero-bytes': 'int', | |
423 | + 'data': {'*status': 'str', '*errmsg': 'str', '*total': 'int', '*dirty': 'int', | |
424 | + '*transferred': 'int', '*zero-bytes': 'int', '*reused': 'int', | |
425 | '*start-time': 'int', '*end-time': 'int', | |
426 | '*backup-file': 'str', '*uuid': 'str' } } | |
427 | ||
817b7667 | 428 | @@ -816,6 +821,8 @@ |
c96a4a38 DM |
429 | # |
430 | # @backup-time: backup timestamp (Unix epoch, required for format 'pbs') | |
431 | # | |
432 | +# @incremental: sync incremental changes since last job (optional for format 'pbs') | |
433 | +# | |
434 | # Returns: the uuid of the backup job | |
435 | # | |
436 | ## | |
817b7667 | 437 | @@ -826,6 +833,7 @@ |
c96a4a38 DM |
438 | '*fingerprint': 'str', |
439 | '*backup-id': 'str', | |
440 | '*backup-time': 'int', | |
441 | + '*incremental': 'bool', | |
442 | '*format': 'BackupFormat', | |
443 | '*config-file': 'str', | |
444 | '*firewall-file': 'str', |