1 From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
2 From: Stefan Reiter <s.reiter@proxmox.com>
3 Date: Wed, 19 Aug 2020 17:02:00 +0200
4 Subject: [PATCH] PVE: add query-pbs-bitmap-info QMP call
6 Returns advanced information about dirty bitmaps used (or not used) for
9 Signed-off-by: Stefan Reiter <s.reiter@proxmox.com>
11 monitor/hmp-cmds.c | 28 ++++++-----
12 pve-backup.c | 117 ++++++++++++++++++++++++++++++++-----------
13 qapi/block-core.json | 56 +++++++++++++++++++++
14 3 files changed, 159 insertions(+), 42 deletions(-)
16 diff --git a/monitor/hmp-cmds.c b/monitor/hmp-cmds.c
17 index 604026bb37..95f4e7f5c1 100644
18 --- a/monitor/hmp-cmds.c
19 +++ b/monitor/hmp-cmds.c
20 @@ -198,6 +198,7 @@ void hmp_info_mice(Monitor *mon, const QDict *qdict)
21 void hmp_info_backup(Monitor *mon, const QDict *qdict)
24 + PBSBitmapInfoList *bitmap_info;
26 info = qmp_query_backup(NULL);
28 @@ -228,26 +229,29 @@ void hmp_info_backup(Monitor *mon, const QDict *qdict)
29 // this should not happen normally
30 monitor_printf(mon, "Total size: %d\n", 0);
32 - bool incremental = false;
33 size_t total_or_dirty = info->total;
34 - if (info->has_transferred) {
35 - if (info->has_dirty && info->dirty) {
36 - if (info->dirty < info->total) {
37 - total_or_dirty = info->dirty;
41 + bitmap_info = qmp_query_pbs_bitmap_info(NULL);
43 + while (bitmap_info) {
44 + monitor_printf(mon, "Drive %s:\n",
45 + bitmap_info->value->drive);
46 + monitor_printf(mon, " bitmap action: %s\n",
47 + PBSBitmapAction_str(bitmap_info->value->action));
48 + monitor_printf(mon, " size: %zd\n",
49 + bitmap_info->value->size);
50 + monitor_printf(mon, " dirty: %zd\n",
51 + bitmap_info->value->dirty);
52 + bitmap_info = bitmap_info->next;
55 - int per = (info->transferred * 100)/total_or_dirty;
57 - monitor_printf(mon, "Backup mode: %s\n", incremental ? "incremental" : "full");
58 + qapi_free_PBSBitmapInfoList(bitmap_info);
60 int zero_per = (info->has_zero_bytes && info->zero_bytes) ?
61 (info->zero_bytes * 100)/info->total : 0;
62 monitor_printf(mon, "Total size: %zd\n", info->total);
63 + int trans_per = (info->transferred * 100)/total_or_dirty;
64 monitor_printf(mon, "Transferred bytes: %zd (%d%%)\n",
65 - info->transferred, per);
66 + info->transferred, trans_per);
67 monitor_printf(mon, "Zero bytes: %zd (%d%%)\n",
68 info->zero_bytes, zero_per);
70 diff --git a/pve-backup.c b/pve-backup.c
71 index 40c2697b37..1e7b92a950 100644
74 @@ -46,6 +46,7 @@ static struct PVEBackupState {
82 @@ -670,7 +671,6 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque)
90 @@ -691,18 +691,33 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque)
94 + qemu_mutex_lock(&backup_state.stat.lock);
95 + backup_state.stat.reused = 0;
97 + /* clear previous backup's bitmap_list */
98 + if (backup_state.stat.bitmap_list) {
99 + GList *bl = backup_state.stat.bitmap_list;
101 + g_free(((PBSBitmapInfo *)bl->data)->drive);
103 + bl = g_list_next(bl);
105 + g_list_free(backup_state.stat.bitmap_list);
106 + backup_state.stat.bitmap_list = NULL;
109 if (format == BACKUP_FORMAT_PBS) {
110 if (!task->has_password) {
111 error_set(task->errp, ERROR_CLASS_GENERIC_ERROR, "missing parameter 'password'");
115 if (!task->has_backup_id) {
116 error_set(task->errp, ERROR_CLASS_GENERIC_ERROR, "missing parameter 'backup-id'");
120 if (!task->has_backup_time) {
121 error_set(task->errp, ERROR_CLASS_GENERIC_ERROR, "missing parameter 'backup-time'");
126 int dump_cb_block_size = PROXMOX_BACKUP_DEFAULT_CHUNK_SIZE; // Hardcoded (4M)
127 @@ -729,12 +744,12 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque)
128 error_set(task->errp, ERROR_CLASS_GENERIC_ERROR,
129 "proxmox_backup_new failed: %s", pbs_err);
130 proxmox_backup_free_error(pbs_err);
135 int connect_result = proxmox_backup_co_connect(pbs, task->errp);
136 if (connect_result < 0)
140 /* register all devices */
142 @@ -745,6 +760,8 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque)
143 di->block_size = dump_cb_block_size;
145 const char *devname = bdrv_get_device_name(di->bs);
146 + PBSBitmapAction action = PBS_BITMAP_ACTION_NOT_USED;
147 + size_t dirty = di->size;
149 BdrvDirtyBitmap *bitmap = bdrv_find_dirty_bitmap(di->bs, PBS_BITMAP_NAME);
150 bool expect_only_dirty = false;
151 @@ -753,49 +770,59 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque)
152 if (bitmap == NULL) {
153 bitmap = bdrv_create_dirty_bitmap(di->bs, dump_cb_block_size, PBS_BITMAP_NAME, task->errp);
158 + action = PBS_BITMAP_ACTION_NEW;
160 expect_only_dirty = proxmox_backup_check_incremental(pbs, devname, di->size) != 0;
163 if (expect_only_dirty) {
164 - dirty += bdrv_get_dirty_count(bitmap);
165 + /* track clean chunks as reused */
166 + dirty = MIN(bdrv_get_dirty_count(bitmap), di->size);
167 + backup_state.stat.reused += di->size - dirty;
168 + action = PBS_BITMAP_ACTION_USED;
170 /* mark entire bitmap as dirty to make full backup */
171 bdrv_set_dirty_bitmap(bitmap, 0, di->size);
173 + if (action != PBS_BITMAP_ACTION_NEW) {
174 + action = PBS_BITMAP_ACTION_INVALID;
181 /* after a full backup the old dirty bitmap is invalid anyway */
182 if (bitmap != NULL) {
183 bdrv_release_dirty_bitmap(bitmap);
184 + action = PBS_BITMAP_ACTION_NOT_USED_REMOVED;
188 int dev_id = proxmox_backup_co_register_image(pbs, devname, di->size, expect_only_dirty, task->errp);
194 if (!(di->target = bdrv_backup_dump_create(dump_cb_block_size, di->size, pvebackup_co_dump_pbs_cb, di, task->errp))) {
201 + PBSBitmapInfo *info = g_malloc(sizeof(*info));
202 + info->drive = g_strdup(devname);
203 + info->action = action;
204 + info->size = di->size;
205 + info->dirty = dirty;
206 + backup_state.stat.bitmap_list = g_list_append(backup_state.stat.bitmap_list, info);
208 } else if (format == BACKUP_FORMAT_VMA) {
211 vmaw = vma_writer_create(task->backup_file, uuid, &local_err);
214 error_propagate(task->errp, local_err);
220 /* register all devices for vma writer */
221 @@ -805,7 +832,7 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque)
224 if (!(di->target = bdrv_backup_dump_create(VMA_CLUSTER_SIZE, di->size, pvebackup_co_dump_vma_cb, di, task->errp))) {
229 const char *devname = bdrv_get_device_name(di->bs);
230 @@ -813,16 +840,14 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque)
231 if (di->dev_id <= 0) {
232 error_set(task->errp, ERROR_CLASS_GENERIC_ERROR,
233 "register_stream failed");
238 } else if (format == BACKUP_FORMAT_DIR) {
241 if (mkdir(task->backup_file, 0640) != 0) {
242 error_setg_errno(task->errp, errno, "can't create directory '%s'\n",
247 backup_dir = task->backup_file;
249 @@ -839,18 +864,18 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque)
250 di->size, flags, false, &local_err);
252 error_propagate(task->errp, local_err);
257 di->target = bdrv_open(di->targetfile, NULL, NULL, flags, &local_err);
259 error_propagate(task->errp, local_err);
265 error_set(task->errp, ERROR_CLASS_GENERIC_ERROR, "unknown backup format");
271 @@ -858,7 +883,7 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque)
272 if (task->has_config_file) {
273 if (pvebackup_co_add_config(task->config_file, config_name, format, backup_dir,
274 vmaw, pbs, task->errp) != 0) {
280 @@ -866,12 +891,11 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque)
281 if (task->has_firewall_file) {
282 if (pvebackup_co_add_config(task->firewall_file, firewall_name, format, backup_dir,
283 vmaw, pbs, task->errp) != 0) {
288 /* initialize global backup_state now */
290 - qemu_mutex_lock(&backup_state.stat.lock);
291 + /* note: 'reused' and 'bitmap_list' are initialized earlier */
293 if (backup_state.stat.error) {
294 error_free(backup_state.stat.error);
295 @@ -891,10 +915,9 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque)
296 char *uuid_str = g_strdup(backup_state.stat.uuid_str);
298 backup_state.stat.total = total;
299 - backup_state.stat.dirty = dirty;
300 + backup_state.stat.dirty = total - backup_state.stat.reused;
301 backup_state.stat.transferred = 0;
302 backup_state.stat.zero_bytes = 0;
303 - backup_state.stat.reused = format == BACKUP_FORMAT_PBS && dirty >= total ? 0 : total - dirty;
305 qemu_mutex_unlock(&backup_state.stat.lock);
307 @@ -911,6 +934,9 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque)
308 task->result = uuid_info;
312 + qemu_mutex_unlock(&backup_state.stat.lock);
317 @@ -1074,10 +1100,41 @@ BackupStatus *qmp_query_backup(Error **errp)
321 +PBSBitmapInfoList *qmp_query_pbs_bitmap_info(Error **errp)
323 + PBSBitmapInfoList *head = NULL, **p_next = &head;
325 + qemu_mutex_lock(&backup_state.stat.lock);
327 + GList *l = backup_state.stat.bitmap_list;
329 + PBSBitmapInfo *info = (PBSBitmapInfo *)l->data;
330 + l = g_list_next(l);
332 + /* clone bitmap info to avoid auto free after QMP marshalling */
333 + PBSBitmapInfo *info_ret = g_malloc0(sizeof(*info_ret));
334 + info_ret->drive = g_strdup(info->drive);
335 + info_ret->action = info->action;
336 + info_ret->size = info->size;
337 + info_ret->dirty = info->dirty;
339 + PBSBitmapInfoList *info_list = g_malloc0(sizeof(*info_list));
340 + info_list->value = info_ret;
342 + *p_next = info_list;
343 + p_next = &info_list->next;
346 + qemu_mutex_unlock(&backup_state.stat.lock);
351 ProxmoxSupportStatus *qmp_query_proxmox_support(Error **errp)
353 ProxmoxSupportStatus *ret = g_malloc0(sizeof(*ret));
354 ret->pbs_library_version = g_strdup(proxmox_backup_qemu_version());
355 ret->pbs_dirty_bitmap = true;
356 + ret->query_bitmap_info = true;
359 diff --git a/qapi/block-core.json b/qapi/block-core.json
360 index e0a0a60354..b368371e8e 100644
361 --- a/qapi/block-core.json
362 +++ b/qapi/block-core.json
363 @@ -876,11 +876,14 @@
364 # @pbs-dirty-bitmap: True if dirty-bitmap-incremental backups to PBS are
367 +# @query-bitmap-info: True if the 'query-pbs-bitmap-info' QMP call is supported.
369 # @pbs-library-version: Running version of libproxmox-backup-qemu0 library.
372 { 'struct': 'ProxmoxSupportStatus',
373 'data': { 'pbs-dirty-bitmap': 'bool',
374 + 'query-bitmap-info': 'bool',
375 'pbs-library-version': 'str' } }
380 { 'command': 'query-proxmox-support', 'returns': 'ProxmoxSupportStatus' }
385 +# An action taken on a dirty-bitmap when a backup job was started.
387 +# @not-used: Bitmap mode was not enabled.
389 +# @not-used-removed: Bitmap mode was not enabled, but a bitmap from a
390 +# previous backup still existed and was removed.
392 +# @new: A new bitmap was attached to the drive for this backup.
394 +# @used: An existing bitmap will be used to only backup changed data.
396 +# @invalid: A bitmap existed, but had to be cleared since it's associated
397 +# base snapshot did not match the base given for the current job or
398 +# the crypt mode has changed.
401 +{ 'enum': 'PBSBitmapAction',
402 + 'data': ['not-used', 'not-used-removed', 'new', 'used', 'invalid'] }
407 +# Contains information about dirty bitmaps used for each drive in a PBS backup.
409 +# @drive: The underlying drive.
411 +# @action: The action that was taken when the backup started.
413 +# @size: The total size of the drive.
415 +# @dirty: How much of the drive is considered dirty and will be backed up,
416 +# or 'size' if everything will be.
419 +{ 'struct': 'PBSBitmapInfo',
420 + 'data': { 'drive': 'str', 'action': 'PBSBitmapAction', 'size': 'int',
424 +# @query-pbs-bitmap-info:
426 +# Returns information about dirty bitmaps used on the most recently started
427 +# backup. Returns nothing when the last backup was not using PBS or if no
428 +# backup occured in this session.
430 +# Returns: @PBSBitmapInfo
433 +{ 'command': 'query-pbs-bitmap-info', 'returns': ['PBSBitmapInfo'] }
436 # @BlockDeviceTimedStats: