]>
Commit | Line | Data |
---|---|---|
f00a720d TL |
1 | From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 |
2 | From: Stefan Reiter <s.reiter@proxmox.com> | |
3 | Date: Wed, 19 Aug 2020 17:02:00 +0200 | |
4 | Subject: [PATCH] PVE: add query-pbs-bitmap-info QMP call | |
5 | ||
6 | Returns advanced information about dirty bitmaps used (or not used) for | |
7 | the latest PBS backup. | |
8 | ||
9 | Signed-off-by: Stefan Reiter <s.reiter@proxmox.com> | |
10 | --- | |
11 | monitor/hmp-cmds.c | 28 ++++++----- | |
12 | pve-backup.c | 117 ++++++++++++++++++++++++++++++++----------- | |
13 | qapi/block-core.json | 57 ++++++++++++++++++++- | |
14 | 3 files changed, 159 insertions(+), 43 deletions(-) | |
15 | ||
16 | diff --git a/monitor/hmp-cmds.c b/monitor/hmp-cmds.c | |
17 | index 4f692c15a2..4717fe7d2c 100644 | |
18 | --- a/monitor/hmp-cmds.c | |
19 | +++ b/monitor/hmp-cmds.c | |
20 | @@ -195,6 +195,7 @@ void hmp_info_mice(Monitor *mon, const QDict *qdict) | |
21 | void hmp_info_backup(Monitor *mon, const QDict *qdict) | |
22 | { | |
23 | BackupStatus *info; | |
24 | + PBSBitmapInfoList *bitmap_info; | |
25 | ||
26 | info = qmp_query_backup(NULL); | |
27 | ||
28 | @@ -225,26 +226,29 @@ void hmp_info_backup(Monitor *mon, const QDict *qdict) | |
29 | // this should not happen normally | |
30 | monitor_printf(mon, "Total size: %d\n", 0); | |
31 | } else { | |
32 | - bool incremental = false; | |
33 | size_t total_or_dirty = info->total; | |
34 | - if (info->has_transferred) { | |
35 | - if (info->has_dirty && info->dirty) { | |
36 | - if (info->dirty < info->total) { | |
37 | - total_or_dirty = info->dirty; | |
38 | - incremental = true; | |
39 | - } | |
40 | - } | |
41 | + bitmap_info = qmp_query_pbs_bitmap_info(NULL); | |
42 | + | |
43 | + while (bitmap_info) { | |
44 | + monitor_printf(mon, "Drive %s:\n", | |
45 | + bitmap_info->value->drive); | |
46 | + monitor_printf(mon, " bitmap action: %s\n", | |
47 | + PBSBitmapAction_str(bitmap_info->value->action)); | |
48 | + monitor_printf(mon, " size: %zd\n", | |
49 | + bitmap_info->value->size); | |
50 | + monitor_printf(mon, " dirty: %zd\n", | |
51 | + bitmap_info->value->dirty); | |
52 | + bitmap_info = bitmap_info->next; | |
53 | } | |
54 | ||
55 | - int per = (info->transferred * 100)/total_or_dirty; | |
56 | - | |
57 | - monitor_printf(mon, "Backup mode: %s\n", incremental ? "incremental" : "full"); | |
58 | + qapi_free_PBSBitmapInfoList(bitmap_info); | |
59 | ||
60 | int zero_per = (info->has_zero_bytes && info->zero_bytes) ? | |
61 | (info->zero_bytes * 100)/info->total : 0; | |
62 | monitor_printf(mon, "Total size: %zd\n", info->total); | |
63 | + int trans_per = (info->transferred * 100)/total_or_dirty; | |
64 | monitor_printf(mon, "Transferred bytes: %zd (%d%%)\n", | |
65 | - info->transferred, per); | |
66 | + info->transferred, trans_per); | |
67 | monitor_printf(mon, "Zero bytes: %zd (%d%%)\n", | |
68 | info->zero_bytes, zero_per); | |
69 | ||
70 | diff --git a/pve-backup.c b/pve-backup.c | |
71 | index 7c99554514..c6d8a51406 100644 | |
72 | --- a/pve-backup.c | |
73 | +++ b/pve-backup.c | |
74 | @@ -46,6 +46,7 @@ static struct PVEBackupState { | |
75 | size_t transferred; | |
76 | size_t reused; | |
77 | size_t zero_bytes; | |
78 | + GList *bitmap_list; | |
79 | } stat; | |
80 | int64_t speed; | |
81 | VmaWriter *vmaw; | |
82 | @@ -674,7 +675,6 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque) | |
83 | } | |
84 | ||
85 | size_t total = 0; | |
86 | - size_t dirty = 0; | |
87 | ||
88 | l = di_list; | |
89 | while (l) { | |
90 | @@ -695,18 +695,33 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque) | |
91 | ||
92 | uuid_generate(uuid); | |
93 | ||
94 | + qemu_mutex_lock(&backup_state.stat.lock); | |
95 | + backup_state.stat.reused = 0; | |
96 | + | |
97 | + /* clear previous backup's bitmap_list */ | |
98 | + if (backup_state.stat.bitmap_list) { | |
99 | + GList *bl = backup_state.stat.bitmap_list; | |
100 | + while (bl) { | |
101 | + g_free(((PBSBitmapInfo *)bl->data)->drive); | |
102 | + g_free(bl->data); | |
103 | + bl = g_list_next(bl); | |
104 | + } | |
105 | + g_list_free(backup_state.stat.bitmap_list); | |
106 | + backup_state.stat.bitmap_list = NULL; | |
107 | + } | |
108 | + | |
109 | if (format == BACKUP_FORMAT_PBS) { | |
110 | if (!task->has_password) { | |
111 | error_set(task->errp, ERROR_CLASS_GENERIC_ERROR, "missing parameter 'password'"); | |
112 | - goto err; | |
113 | + goto err_mutex; | |
114 | } | |
115 | if (!task->has_backup_id) { | |
116 | error_set(task->errp, ERROR_CLASS_GENERIC_ERROR, "missing parameter 'backup-id'"); | |
117 | - goto err; | |
118 | + goto err_mutex; | |
119 | } | |
120 | if (!task->has_backup_time) { | |
121 | error_set(task->errp, ERROR_CLASS_GENERIC_ERROR, "missing parameter 'backup-time'"); | |
122 | - goto err; | |
123 | + goto err_mutex; | |
124 | } | |
125 | ||
126 | int dump_cb_block_size = PROXMOX_BACKUP_DEFAULT_CHUNK_SIZE; // Hardcoded (4M) | |
127 | @@ -733,12 +748,12 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque) | |
128 | error_set(task->errp, ERROR_CLASS_GENERIC_ERROR, | |
129 | "proxmox_backup_new failed: %s", pbs_err); | |
130 | proxmox_backup_free_error(pbs_err); | |
131 | - goto err; | |
132 | + goto err_mutex; | |
133 | } | |
134 | ||
135 | int connect_result = proxmox_backup_co_connect(pbs, task->errp); | |
136 | if (connect_result < 0) | |
137 | - goto err; | |
138 | + goto err_mutex; | |
139 | ||
140 | /* register all devices */ | |
141 | l = di_list; | |
142 | @@ -749,6 +764,8 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque) | |
143 | di->block_size = dump_cb_block_size; | |
144 | ||
145 | const char *devname = bdrv_get_device_name(di->bs); | |
146 | + PBSBitmapAction action = PBS_BITMAP_ACTION_NOT_USED; | |
147 | + size_t dirty = di->size; | |
148 | ||
149 | BdrvDirtyBitmap *bitmap = bdrv_find_dirty_bitmap(di->bs, PBS_BITMAP_NAME); | |
150 | bool expect_only_dirty = false; | |
151 | @@ -757,49 +774,59 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque) | |
152 | if (bitmap == NULL) { | |
153 | bitmap = bdrv_create_dirty_bitmap(di->bs, dump_cb_block_size, PBS_BITMAP_NAME, task->errp); | |
154 | if (!bitmap) { | |
155 | - goto err; | |
156 | + goto err_mutex; | |
157 | } | |
158 | + action = PBS_BITMAP_ACTION_NEW; | |
159 | } else { | |
160 | expect_only_dirty = proxmox_backup_check_incremental(pbs, devname, di->size) != 0; | |
161 | } | |
162 | ||
163 | if (expect_only_dirty) { | |
164 | - dirty += bdrv_get_dirty_count(bitmap); | |
165 | + /* track clean chunks as reused */ | |
166 | + dirty = MIN(bdrv_get_dirty_count(bitmap), di->size); | |
167 | + backup_state.stat.reused += di->size - dirty; | |
168 | + action = PBS_BITMAP_ACTION_USED; | |
169 | } else { | |
170 | /* mark entire bitmap as dirty to make full backup */ | |
171 | bdrv_set_dirty_bitmap(bitmap, 0, di->size); | |
172 | - dirty += di->size; | |
173 | + if (action != PBS_BITMAP_ACTION_NEW) { | |
174 | + action = PBS_BITMAP_ACTION_INVALID; | |
175 | + } | |
176 | } | |
177 | di->bitmap = bitmap; | |
178 | } else { | |
179 | - dirty += di->size; | |
180 | - | |
181 | /* after a full backup the old dirty bitmap is invalid anyway */ | |
182 | if (bitmap != NULL) { | |
183 | bdrv_release_dirty_bitmap(bitmap); | |
184 | + action = PBS_BITMAP_ACTION_NOT_USED_REMOVED; | |
185 | } | |
186 | } | |
187 | ||
188 | int dev_id = proxmox_backup_co_register_image(pbs, devname, di->size, expect_only_dirty, task->errp); | |
189 | if (dev_id < 0) { | |
190 | - goto err; | |
191 | + goto err_mutex; | |
192 | } | |
193 | ||
194 | if (!(di->target = bdrv_backup_dump_create(dump_cb_block_size, di->size, pvebackup_co_dump_pbs_cb, di, task->errp))) { | |
195 | - goto err; | |
196 | + goto err_mutex; | |
197 | } | |
198 | ||
199 | di->dev_id = dev_id; | |
200 | + | |
201 | + PBSBitmapInfo *info = g_malloc(sizeof(*info)); | |
202 | + info->drive = g_strdup(devname); | |
203 | + info->action = action; | |
204 | + info->size = di->size; | |
205 | + info->dirty = dirty; | |
206 | + backup_state.stat.bitmap_list = g_list_append(backup_state.stat.bitmap_list, info); | |
207 | } | |
208 | } else if (format == BACKUP_FORMAT_VMA) { | |
209 | - dirty = total; | |
210 | - | |
211 | vmaw = vma_writer_create(task->backup_file, uuid, &local_err); | |
212 | if (!vmaw) { | |
213 | if (local_err) { | |
214 | error_propagate(task->errp, local_err); | |
215 | } | |
216 | - goto err; | |
217 | + goto err_mutex; | |
218 | } | |
219 | ||
220 | /* register all devices for vma writer */ | |
221 | @@ -809,7 +836,7 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque) | |
222 | l = g_list_next(l); | |
223 | ||
224 | if (!(di->target = bdrv_backup_dump_create(VMA_CLUSTER_SIZE, di->size, pvebackup_co_dump_vma_cb, di, task->errp))) { | |
225 | - goto err; | |
226 | + goto err_mutex; | |
227 | } | |
228 | ||
229 | const char *devname = bdrv_get_device_name(di->bs); | |
230 | @@ -817,16 +844,14 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque) | |
231 | if (di->dev_id <= 0) { | |
232 | error_set(task->errp, ERROR_CLASS_GENERIC_ERROR, | |
233 | "register_stream failed"); | |
234 | - goto err; | |
235 | + goto err_mutex; | |
236 | } | |
237 | } | |
238 | } else if (format == BACKUP_FORMAT_DIR) { | |
239 | - dirty = total; | |
240 | - | |
241 | if (mkdir(task->backup_file, 0640) != 0) { | |
242 | error_setg_errno(task->errp, errno, "can't create directory '%s'\n", | |
243 | task->backup_file); | |
244 | - goto err; | |
245 | + goto err_mutex; | |
246 | } | |
247 | backup_dir = task->backup_file; | |
248 | ||
249 | @@ -843,18 +868,18 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque) | |
250 | di->size, flags, false, &local_err); | |
251 | if (local_err) { | |
252 | error_propagate(task->errp, local_err); | |
253 | - goto err; | |
254 | + goto err_mutex; | |
255 | } | |
256 | ||
257 | di->target = bdrv_open(di->targetfile, NULL, NULL, flags, &local_err); | |
258 | if (!di->target) { | |
259 | error_propagate(task->errp, local_err); | |
260 | - goto err; | |
261 | + goto err_mutex; | |
262 | } | |
263 | } | |
264 | } else { | |
265 | error_set(task->errp, ERROR_CLASS_GENERIC_ERROR, "unknown backup format"); | |
266 | - goto err; | |
267 | + goto err_mutex; | |
268 | } | |
269 | ||
270 | ||
271 | @@ -862,7 +887,7 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque) | |
272 | if (task->has_config_file) { | |
273 | if (pvebackup_co_add_config(task->config_file, config_name, format, backup_dir, | |
274 | vmaw, pbs, task->errp) != 0) { | |
275 | - goto err; | |
276 | + goto err_mutex; | |
277 | } | |
278 | } | |
279 | ||
280 | @@ -870,12 +895,11 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque) | |
281 | if (task->has_firewall_file) { | |
282 | if (pvebackup_co_add_config(task->firewall_file, firewall_name, format, backup_dir, | |
283 | vmaw, pbs, task->errp) != 0) { | |
284 | - goto err; | |
285 | + goto err_mutex; | |
286 | } | |
287 | } | |
288 | /* initialize global backup_state now */ | |
289 | - | |
290 | - qemu_mutex_lock(&backup_state.stat.lock); | |
291 | + /* note: 'reused' and 'bitmap_list' are initialized earlier */ | |
292 | ||
293 | if (backup_state.stat.error) { | |
294 | error_free(backup_state.stat.error); | |
295 | @@ -895,10 +919,9 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque) | |
296 | char *uuid_str = g_strdup(backup_state.stat.uuid_str); | |
297 | ||
298 | backup_state.stat.total = total; | |
299 | - backup_state.stat.dirty = dirty; | |
300 | + backup_state.stat.dirty = total - backup_state.stat.reused; | |
301 | backup_state.stat.transferred = 0; | |
302 | backup_state.stat.zero_bytes = 0; | |
303 | - backup_state.stat.reused = format == BACKUP_FORMAT_PBS && dirty >= total ? 0 : total - dirty; | |
304 | ||
305 | qemu_mutex_unlock(&backup_state.stat.lock); | |
306 | ||
307 | @@ -915,6 +938,9 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque) | |
308 | task->result = uuid_info; | |
309 | return; | |
310 | ||
311 | +err_mutex: | |
312 | + qemu_mutex_unlock(&backup_state.stat.lock); | |
313 | + | |
314 | err: | |
315 | ||
316 | l = di_list; | |
317 | @@ -1078,9 +1104,40 @@ BackupStatus *qmp_query_backup(Error **errp) | |
318 | return info; | |
319 | } | |
320 | ||
321 | +PBSBitmapInfoList *qmp_query_pbs_bitmap_info(Error **errp) | |
322 | +{ | |
323 | + PBSBitmapInfoList *head = NULL, **p_next = &head; | |
324 | + | |
325 | + qemu_mutex_lock(&backup_state.stat.lock); | |
326 | + | |
327 | + GList *l = backup_state.stat.bitmap_list; | |
328 | + while (l) { | |
329 | + PBSBitmapInfo *info = (PBSBitmapInfo *)l->data; | |
330 | + l = g_list_next(l); | |
331 | + | |
332 | + /* clone bitmap info to avoid auto free after QMP marshalling */ | |
333 | + PBSBitmapInfo *info_ret = g_malloc0(sizeof(*info_ret)); | |
334 | + info_ret->drive = g_strdup(info->drive); | |
335 | + info_ret->action = info->action; | |
336 | + info_ret->size = info->size; | |
337 | + info_ret->dirty = info->dirty; | |
338 | + | |
339 | + PBSBitmapInfoList *info_list = g_malloc0(sizeof(*info_list)); | |
340 | + info_list->value = info_ret; | |
341 | + | |
342 | + *p_next = info_list; | |
343 | + p_next = &info_list->next; | |
344 | + } | |
345 | + | |
346 | + qemu_mutex_unlock(&backup_state.stat.lock); | |
347 | + | |
348 | + return head; | |
349 | +} | |
350 | + | |
351 | ProxmoxSupportStatus *qmp_query_proxmox_support(Error **errp) | |
352 | { | |
353 | ProxmoxSupportStatus *ret = g_malloc0(sizeof(*ret)); | |
354 | ret->pbs_dirty_bitmap = true; | |
355 | + ret->query_bitmap_info = true; | |
356 | return ret; | |
357 | } | |
358 | diff --git a/qapi/block-core.json b/qapi/block-core.json | |
359 | index 355a184ea8..a3d54b85e2 100644 | |
360 | --- a/qapi/block-core.json | |
361 | +++ b/qapi/block-core.json | |
362 | @@ -875,9 +875,11 @@ | |
363 | # @pbs-dirty-bitmap: True if dirty-bitmap-incremental backups to PBS are | |
364 | # supported. | |
365 | # | |
366 | +# @query-bitmap-info: True if the 'query-pbs-bitmap-info' QMP call is supported. | |
367 | +# | |
368 | ## | |
369 | { 'struct': 'ProxmoxSupportStatus', | |
370 | - 'data': { 'pbs-dirty-bitmap': 'bool' } } | |
371 | + 'data': { 'pbs-dirty-bitmap': 'bool', 'query-bitmap-info': 'bool' } } | |
372 | ||
373 | ## | |
374 | # @query-proxmox-support: | |
375 | @@ -889,6 +891,59 @@ | |
376 | ## | |
377 | { 'command': 'query-proxmox-support', 'returns': 'ProxmoxSupportStatus' } | |
378 | ||
379 | +## | |
380 | +# @PBSBitmapAction: | |
381 | +# | |
382 | +# An action taken on a dirty-bitmap when a backup job was started. | |
383 | +# | |
384 | +# @not-used: Bitmap mode was not enabled. | |
385 | +# | |
386 | +# @not-used-removed: Bitmap mode was not enabled, but a bitmap from a | |
387 | +# previous backup still existed and was removed. | |
388 | +# | |
389 | +# @new: A new bitmap was attached to the drive for this backup. | |
390 | +# | |
391 | +# @used: An existing bitmap will be used to only backup changed data. | |
392 | +# | |
393 | +# @invalid: A bitmap existed, but had to be cleared since it's associated | |
394 | +# base snapshot did not match the base given for the current job or | |
395 | +# the crypt mode has changed. | |
396 | +# | |
397 | +## | |
398 | +{ 'enum': 'PBSBitmapAction', | |
399 | + 'data': ['not-used', 'not-used-removed', 'new', 'used', 'invalid'] } | |
400 | + | |
401 | +## | |
402 | +# @PBSBitmapInfo: | |
403 | +# | |
404 | +# Contains information about dirty bitmaps used for each drive in a PBS backup. | |
405 | +# | |
406 | +# @drive: The underlying drive. | |
407 | +# | |
408 | +# @action: The action that was taken when the backup started. | |
409 | +# | |
410 | +# @size: The total size of the drive. | |
411 | +# | |
412 | +# @dirty: How much of the drive is considered dirty and will be backed up, | |
413 | +# or 'size' if everything will be. | |
414 | +# | |
415 | +## | |
416 | +{ 'struct': 'PBSBitmapInfo', | |
417 | + 'data': { 'drive': 'str', 'action': 'PBSBitmapAction', 'size': 'int', | |
418 | + 'dirty': 'int' } } | |
419 | + | |
420 | +## | |
421 | +# @query-pbs-bitmap-info: | |
422 | +# | |
423 | +# Returns information about dirty bitmaps used on the most recently started | |
424 | +# backup. Returns nothing when the last backup was not using PBS or if no | |
425 | +# backup occured in this session. | |
426 | +# | |
427 | +# Returns: @PBSBitmapInfo | |
428 | +# | |
429 | +## | |
430 | +{ 'command': 'query-pbs-bitmap-info', 'returns': ['PBSBitmapInfo'] } | |
431 | + | |
432 | ## | |
433 | # @BlockDeviceTimedStats: | |
434 | # |