1 From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
2 From: Stefan Reiter <s.reiter@proxmox.com>
3 Date: Wed, 19 Aug 2020 17:02:00 +0200
4 Subject: [PATCH] PVE: add query-pbs-bitmap-info QMP call
6 Returns advanced information about dirty bitmaps used (or not used) for
9 Signed-off-by: Stefan Reiter <s.reiter@proxmox.com>
10 Signed-off-by: Thomas Lamprecht <t.lamprecht@proxmox.com>
12 monitor/hmp-cmds.c | 28 ++++++-----
13 pve-backup.c | 117 ++++++++++++++++++++++++++++++++-----------
14 qapi/block-core.json | 56 +++++++++++++++++++++
15 3 files changed, 159 insertions(+), 42 deletions(-)
17 diff --git a/monitor/hmp-cmds.c b/monitor/hmp-cmds.c
18 index 087161a967..9a67e544ce 100644
19 --- a/monitor/hmp-cmds.c
20 +++ b/monitor/hmp-cmds.c
21 @@ -148,6 +148,7 @@ void hmp_sync_profile(Monitor *mon, const QDict *qdict)
22 void hmp_info_backup(Monitor *mon, const QDict *qdict)
25 + PBSBitmapInfoList *bitmap_info;
27 info = qmp_query_backup(NULL);
29 @@ -178,26 +179,29 @@ void hmp_info_backup(Monitor *mon, const QDict *qdict)
30 // this should not happen normally
31 monitor_printf(mon, "Total size: %d\n", 0);
33 - bool incremental = false;
34 size_t total_or_dirty = info->total;
35 - if (info->has_transferred) {
36 - if (info->has_dirty && info->dirty) {
37 - if (info->dirty < info->total) {
38 - total_or_dirty = info->dirty;
42 + bitmap_info = qmp_query_pbs_bitmap_info(NULL);
44 + while (bitmap_info) {
45 + monitor_printf(mon, "Drive %s:\n",
46 + bitmap_info->value->drive);
47 + monitor_printf(mon, " bitmap action: %s\n",
48 + PBSBitmapAction_str(bitmap_info->value->action));
49 + monitor_printf(mon, " size: %zd\n",
50 + bitmap_info->value->size);
51 + monitor_printf(mon, " dirty: %zd\n",
52 + bitmap_info->value->dirty);
53 + bitmap_info = bitmap_info->next;
56 - int per = (info->transferred * 100)/total_or_dirty;
58 - monitor_printf(mon, "Backup mode: %s\n", incremental ? "incremental" : "full");
59 + qapi_free_PBSBitmapInfoList(bitmap_info);
61 int zero_per = (info->has_zero_bytes && info->zero_bytes) ?
62 (info->zero_bytes * 100)/info->total : 0;
63 monitor_printf(mon, "Total size: %zd\n", info->total);
64 + int trans_per = (info->transferred * 100)/total_or_dirty;
65 monitor_printf(mon, "Transferred bytes: %zd (%d%%)\n",
66 - info->transferred, per);
67 + info->transferred, trans_per);
68 monitor_printf(mon, "Zero bytes: %zd (%d%%)\n",
69 info->zero_bytes, zero_per);
71 diff --git a/pve-backup.c b/pve-backup.c
72 index 9eb8645e63..2db35f90e0 100644
75 @@ -48,6 +48,7 @@ static struct PVEBackupState {
83 @@ -663,7 +664,6 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque)
91 @@ -684,18 +684,33 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque)
95 + qemu_mutex_lock(&backup_state.stat.lock);
96 + backup_state.stat.reused = 0;
98 + /* clear previous backup's bitmap_list */
99 + if (backup_state.stat.bitmap_list) {
100 + GList *bl = backup_state.stat.bitmap_list;
102 + g_free(((PBSBitmapInfo *)bl->data)->drive);
104 + bl = g_list_next(bl);
106 + g_list_free(backup_state.stat.bitmap_list);
107 + backup_state.stat.bitmap_list = NULL;
110 if (format == BACKUP_FORMAT_PBS) {
111 if (!task->password) {
112 error_set(task->errp, ERROR_CLASS_GENERIC_ERROR, "missing parameter 'password'");
116 if (!task->backup_id) {
117 error_set(task->errp, ERROR_CLASS_GENERIC_ERROR, "missing parameter 'backup-id'");
121 if (!task->has_backup_time) {
122 error_set(task->errp, ERROR_CLASS_GENERIC_ERROR, "missing parameter 'backup-time'");
127 int dump_cb_block_size = PROXMOX_BACKUP_DEFAULT_CHUNK_SIZE; // Hardcoded (4M)
128 @@ -722,12 +737,12 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque)
129 error_set(task->errp, ERROR_CLASS_GENERIC_ERROR,
130 "proxmox_backup_new failed: %s", pbs_err);
131 proxmox_backup_free_error(pbs_err);
136 int connect_result = proxmox_backup_co_connect(pbs, task->errp);
137 if (connect_result < 0)
141 /* register all devices */
143 @@ -738,6 +753,8 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque)
144 di->block_size = dump_cb_block_size;
146 const char *devname = bdrv_get_device_name(di->bs);
147 + PBSBitmapAction action = PBS_BITMAP_ACTION_NOT_USED;
148 + size_t dirty = di->size;
150 BdrvDirtyBitmap *bitmap = bdrv_find_dirty_bitmap(di->bs, PBS_BITMAP_NAME);
151 bool expect_only_dirty = false;
152 @@ -746,49 +763,59 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque)
153 if (bitmap == NULL) {
154 bitmap = bdrv_create_dirty_bitmap(di->bs, dump_cb_block_size, PBS_BITMAP_NAME, task->errp);
159 + action = PBS_BITMAP_ACTION_NEW;
161 expect_only_dirty = proxmox_backup_check_incremental(pbs, devname, di->size) != 0;
164 if (expect_only_dirty) {
165 - dirty += bdrv_get_dirty_count(bitmap);
166 + /* track clean chunks as reused */
167 + dirty = MIN(bdrv_get_dirty_count(bitmap), di->size);
168 + backup_state.stat.reused += di->size - dirty;
169 + action = PBS_BITMAP_ACTION_USED;
171 /* mark entire bitmap as dirty to make full backup */
172 bdrv_set_dirty_bitmap(bitmap, 0, di->size);
174 + if (action != PBS_BITMAP_ACTION_NEW) {
175 + action = PBS_BITMAP_ACTION_INVALID;
182 /* after a full backup the old dirty bitmap is invalid anyway */
183 if (bitmap != NULL) {
184 bdrv_release_dirty_bitmap(bitmap);
185 + action = PBS_BITMAP_ACTION_NOT_USED_REMOVED;
189 int dev_id = proxmox_backup_co_register_image(pbs, devname, di->size, expect_only_dirty, task->errp);
195 if (!(di->target = bdrv_backup_dump_create(dump_cb_block_size, di->size, pvebackup_co_dump_pbs_cb, di, task->errp))) {
202 + PBSBitmapInfo *info = g_malloc(sizeof(*info));
203 + info->drive = g_strdup(devname);
204 + info->action = action;
205 + info->size = di->size;
206 + info->dirty = dirty;
207 + backup_state.stat.bitmap_list = g_list_append(backup_state.stat.bitmap_list, info);
209 } else if (format == BACKUP_FORMAT_VMA) {
212 vmaw = vma_writer_create(task->backup_file, uuid, &local_err);
215 error_propagate(task->errp, local_err);
221 /* register all devices for vma writer */
222 @@ -798,7 +825,7 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque)
225 if (!(di->target = bdrv_backup_dump_create(VMA_CLUSTER_SIZE, di->size, pvebackup_co_dump_vma_cb, di, task->errp))) {
230 const char *devname = bdrv_get_device_name(di->bs);
231 @@ -806,16 +833,14 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque)
232 if (di->dev_id <= 0) {
233 error_set(task->errp, ERROR_CLASS_GENERIC_ERROR,
234 "register_stream failed");
239 } else if (format == BACKUP_FORMAT_DIR) {
242 if (mkdir(task->backup_file, 0640) != 0) {
243 error_setg_errno(task->errp, errno, "can't create directory '%s'\n",
248 backup_dir = task->backup_file;
250 @@ -832,18 +857,18 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque)
251 di->size, flags, false, &local_err);
253 error_propagate(task->errp, local_err);
258 di->target = bdrv_open(di->targetfile, NULL, NULL, flags, &local_err);
260 error_propagate(task->errp, local_err);
266 error_set(task->errp, ERROR_CLASS_GENERIC_ERROR, "unknown backup format");
272 @@ -851,7 +876,7 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque)
273 if (task->config_file) {
274 if (pvebackup_co_add_config(task->config_file, config_name, format, backup_dir,
275 vmaw, pbs, task->errp) != 0) {
281 @@ -859,12 +884,11 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque)
282 if (task->firewall_file) {
283 if (pvebackup_co_add_config(task->firewall_file, firewall_name, format, backup_dir,
284 vmaw, pbs, task->errp) != 0) {
289 /* initialize global backup_state now */
291 - qemu_mutex_lock(&backup_state.stat.lock);
292 + /* note: 'reused' and 'bitmap_list' are initialized earlier */
294 if (backup_state.stat.error) {
295 error_free(backup_state.stat.error);
296 @@ -884,10 +908,9 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque)
297 char *uuid_str = g_strdup(backup_state.stat.uuid_str);
299 backup_state.stat.total = total;
300 - backup_state.stat.dirty = dirty;
301 + backup_state.stat.dirty = total - backup_state.stat.reused;
302 backup_state.stat.transferred = 0;
303 backup_state.stat.zero_bytes = 0;
304 - backup_state.stat.reused = format == BACKUP_FORMAT_PBS && dirty >= total ? 0 : total - dirty;
306 qemu_mutex_unlock(&backup_state.stat.lock);
308 @@ -904,6 +927,9 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque)
309 task->result = uuid_info;
313 + qemu_mutex_unlock(&backup_state.stat.lock);
318 @@ -1055,11 +1081,42 @@ BackupStatus *qmp_query_backup(Error **errp)
322 +PBSBitmapInfoList *qmp_query_pbs_bitmap_info(Error **errp)
324 + PBSBitmapInfoList *head = NULL, **p_next = &head;
326 + qemu_mutex_lock(&backup_state.stat.lock);
328 + GList *l = backup_state.stat.bitmap_list;
330 + PBSBitmapInfo *info = (PBSBitmapInfo *)l->data;
331 + l = g_list_next(l);
333 + /* clone bitmap info to avoid auto free after QMP marshalling */
334 + PBSBitmapInfo *info_ret = g_malloc0(sizeof(*info_ret));
335 + info_ret->drive = g_strdup(info->drive);
336 + info_ret->action = info->action;
337 + info_ret->size = info->size;
338 + info_ret->dirty = info->dirty;
340 + PBSBitmapInfoList *info_list = g_malloc0(sizeof(*info_list));
341 + info_list->value = info_ret;
343 + *p_next = info_list;
344 + p_next = &info_list->next;
347 + qemu_mutex_unlock(&backup_state.stat.lock);
352 ProxmoxSupportStatus *qmp_query_proxmox_support(Error **errp)
354 ProxmoxSupportStatus *ret = g_malloc0(sizeof(*ret));
355 ret->pbs_library_version = g_strdup(proxmox_backup_qemu_version());
356 ret->pbs_dirty_bitmap = true;
357 ret->pbs_dirty_bitmap_savevm = true;
358 + ret->query_bitmap_info = true;
361 diff --git a/qapi/block-core.json b/qapi/block-core.json
362 index 1ac535fcf2..130d5f065f 100644
363 --- a/qapi/block-core.json
364 +++ b/qapi/block-core.json
366 # @pbs-dirty-bitmap: True if dirty-bitmap-incremental backups to PBS are
369 +# @query-bitmap-info: True if the 'query-pbs-bitmap-info' QMP call is supported.
371 # @pbs-dirty-bitmap-savevm: True if 'dirty-bitmaps' migration capability can
372 # safely be set for savevm-async.
376 { 'struct': 'ProxmoxSupportStatus',
377 'data': { 'pbs-dirty-bitmap': 'bool',
378 + 'query-bitmap-info': 'bool',
379 'pbs-dirty-bitmap-savevm': 'bool',
380 'pbs-library-version': 'str' } }
384 { 'command': 'query-proxmox-support', 'returns': 'ProxmoxSupportStatus' }
389 +# An action taken on a dirty-bitmap when a backup job was started.
391 +# @not-used: Bitmap mode was not enabled.
393 +# @not-used-removed: Bitmap mode was not enabled, but a bitmap from a
394 +# previous backup still existed and was removed.
396 +# @new: A new bitmap was attached to the drive for this backup.
398 +# @used: An existing bitmap will be used to only backup changed data.
400 +# @invalid: A bitmap existed, but had to be cleared since it's associated
401 +# base snapshot did not match the base given for the current job or
402 +# the crypt mode has changed.
405 +{ 'enum': 'PBSBitmapAction',
406 + 'data': ['not-used', 'not-used-removed', 'new', 'used', 'invalid'] }
411 +# Contains information about dirty bitmaps used for each drive in a PBS backup.
413 +# @drive: The underlying drive.
415 +# @action: The action that was taken when the backup started.
417 +# @size: The total size of the drive.
419 +# @dirty: How much of the drive is considered dirty and will be backed up,
420 +# or 'size' if everything will be.
423 +{ 'struct': 'PBSBitmapInfo',
424 + 'data': { 'drive': 'str', 'action': 'PBSBitmapAction', 'size': 'int',
428 +# @query-pbs-bitmap-info:
430 +# Returns information about dirty bitmaps used on the most recently started
431 +# backup. Returns nothing when the last backup was not using PBS or if no
432 +# backup occured in this session.
434 +# Returns: @PBSBitmapInfo
437 +{ 'command': 'query-pbs-bitmap-info', 'returns': ['PBSBitmapInfo'] }
440 # @BlockDeviceTimedStats: