]>
Commit | Line | Data |
---|---|---|
f00a720d TL |
1 | From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 |
2 | From: Stefan Reiter <s.reiter@proxmox.com> | |
3 | Date: Wed, 19 Aug 2020 17:02:00 +0200 | |
4 | Subject: [PATCH] PVE: add query-pbs-bitmap-info QMP call | |
5 | ||
6 | Returns advanced information about dirty bitmaps used (or not used) for | |
7 | the latest PBS backup. | |
8 | ||
9 | Signed-off-by: Stefan Reiter <s.reiter@proxmox.com> | |
ddbf7a87 | 10 | Signed-off-by: Thomas Lamprecht <t.lamprecht@proxmox.com> |
f00a720d TL |
11 | --- |
12 | monitor/hmp-cmds.c | 28 ++++++----- | |
13 | pve-backup.c | 117 ++++++++++++++++++++++++++++++++----------- | |
32ee4115 SR |
14 | qapi/block-core.json | 56 +++++++++++++++++++++ |
15 | 3 files changed, 159 insertions(+), 42 deletions(-) | |
f00a720d TL |
16 | |
17 | diff --git a/monitor/hmp-cmds.c b/monitor/hmp-cmds.c | |
5b15e2ec | 18 | index 4c1671e289..c1152f55a7 100644 |
f00a720d TL |
19 | --- a/monitor/hmp-cmds.c |
20 | +++ b/monitor/hmp-cmds.c | |
5b15e2ec | 21 | @@ -200,6 +200,7 @@ void hmp_info_mice(Monitor *mon, const QDict *qdict) |
f00a720d TL |
22 | void hmp_info_backup(Monitor *mon, const QDict *qdict) |
23 | { | |
24 | BackupStatus *info; | |
25 | + PBSBitmapInfoList *bitmap_info; | |
26 | ||
27 | info = qmp_query_backup(NULL); | |
28 | ||
5b15e2ec | 29 | @@ -230,26 +231,29 @@ void hmp_info_backup(Monitor *mon, const QDict *qdict) |
f00a720d TL |
30 | // this should not happen normally |
31 | monitor_printf(mon, "Total size: %d\n", 0); | |
32 | } else { | |
33 | - bool incremental = false; | |
34 | size_t total_or_dirty = info->total; | |
35 | - if (info->has_transferred) { | |
36 | - if (info->has_dirty && info->dirty) { | |
37 | - if (info->dirty < info->total) { | |
38 | - total_or_dirty = info->dirty; | |
39 | - incremental = true; | |
40 | - } | |
41 | - } | |
42 | + bitmap_info = qmp_query_pbs_bitmap_info(NULL); | |
43 | + | |
44 | + while (bitmap_info) { | |
45 | + monitor_printf(mon, "Drive %s:\n", | |
46 | + bitmap_info->value->drive); | |
47 | + monitor_printf(mon, " bitmap action: %s\n", | |
48 | + PBSBitmapAction_str(bitmap_info->value->action)); | |
49 | + monitor_printf(mon, " size: %zd\n", | |
50 | + bitmap_info->value->size); | |
51 | + monitor_printf(mon, " dirty: %zd\n", | |
52 | + bitmap_info->value->dirty); | |
53 | + bitmap_info = bitmap_info->next; | |
54 | } | |
55 | ||
56 | - int per = (info->transferred * 100)/total_or_dirty; | |
57 | - | |
58 | - monitor_printf(mon, "Backup mode: %s\n", incremental ? "incremental" : "full"); | |
59 | + qapi_free_PBSBitmapInfoList(bitmap_info); | |
60 | ||
61 | int zero_per = (info->has_zero_bytes && info->zero_bytes) ? | |
62 | (info->zero_bytes * 100)/info->total : 0; | |
63 | monitor_printf(mon, "Total size: %zd\n", info->total); | |
64 | + int trans_per = (info->transferred * 100)/total_or_dirty; | |
65 | monitor_printf(mon, "Transferred bytes: %zd (%d%%)\n", | |
66 | - info->transferred, per); | |
67 | + info->transferred, trans_per); | |
68 | monitor_printf(mon, "Zero bytes: %zd (%d%%)\n", | |
69 | info->zero_bytes, zero_per); | |
70 | ||
71 | diff --git a/pve-backup.c b/pve-backup.c | |
4567474e | 72 | index 4684789813..f90abaa50a 100644 |
f00a720d TL |
73 | --- a/pve-backup.c |
74 | +++ b/pve-backup.c | |
75 | @@ -46,6 +46,7 @@ static struct PVEBackupState { | |
76 | size_t transferred; | |
77 | size_t reused; | |
78 | size_t zero_bytes; | |
79 | + GList *bitmap_list; | |
80 | } stat; | |
81 | int64_t speed; | |
82 | VmaWriter *vmaw; | |
8dca018b | 83 | @@ -672,7 +673,6 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque) |
f00a720d TL |
84 | } |
85 | ||
86 | size_t total = 0; | |
87 | - size_t dirty = 0; | |
88 | ||
89 | l = di_list; | |
90 | while (l) { | |
8dca018b | 91 | @@ -693,18 +693,33 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque) |
f00a720d TL |
92 | |
93 | uuid_generate(uuid); | |
94 | ||
95 | + qemu_mutex_lock(&backup_state.stat.lock); | |
96 | + backup_state.stat.reused = 0; | |
97 | + | |
98 | + /* clear previous backup's bitmap_list */ | |
99 | + if (backup_state.stat.bitmap_list) { | |
100 | + GList *bl = backup_state.stat.bitmap_list; | |
101 | + while (bl) { | |
102 | + g_free(((PBSBitmapInfo *)bl->data)->drive); | |
103 | + g_free(bl->data); | |
104 | + bl = g_list_next(bl); | |
105 | + } | |
106 | + g_list_free(backup_state.stat.bitmap_list); | |
107 | + backup_state.stat.bitmap_list = NULL; | |
108 | + } | |
109 | + | |
110 | if (format == BACKUP_FORMAT_PBS) { | |
111 | if (!task->has_password) { | |
112 | error_set(task->errp, ERROR_CLASS_GENERIC_ERROR, "missing parameter 'password'"); | |
113 | - goto err; | |
114 | + goto err_mutex; | |
115 | } | |
116 | if (!task->has_backup_id) { | |
117 | error_set(task->errp, ERROR_CLASS_GENERIC_ERROR, "missing parameter 'backup-id'"); | |
118 | - goto err; | |
119 | + goto err_mutex; | |
120 | } | |
121 | if (!task->has_backup_time) { | |
122 | error_set(task->errp, ERROR_CLASS_GENERIC_ERROR, "missing parameter 'backup-time'"); | |
123 | - goto err; | |
124 | + goto err_mutex; | |
125 | } | |
126 | ||
127 | int dump_cb_block_size = PROXMOX_BACKUP_DEFAULT_CHUNK_SIZE; // Hardcoded (4M) | |
8dca018b | 128 | @@ -731,12 +746,12 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque) |
f00a720d TL |
129 | error_set(task->errp, ERROR_CLASS_GENERIC_ERROR, |
130 | "proxmox_backup_new failed: %s", pbs_err); | |
131 | proxmox_backup_free_error(pbs_err); | |
132 | - goto err; | |
133 | + goto err_mutex; | |
134 | } | |
135 | ||
136 | int connect_result = proxmox_backup_co_connect(pbs, task->errp); | |
137 | if (connect_result < 0) | |
138 | - goto err; | |
139 | + goto err_mutex; | |
140 | ||
141 | /* register all devices */ | |
142 | l = di_list; | |
8dca018b | 143 | @@ -747,6 +762,8 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque) |
f00a720d TL |
144 | di->block_size = dump_cb_block_size; |
145 | ||
146 | const char *devname = bdrv_get_device_name(di->bs); | |
147 | + PBSBitmapAction action = PBS_BITMAP_ACTION_NOT_USED; | |
148 | + size_t dirty = di->size; | |
149 | ||
150 | BdrvDirtyBitmap *bitmap = bdrv_find_dirty_bitmap(di->bs, PBS_BITMAP_NAME); | |
151 | bool expect_only_dirty = false; | |
8dca018b | 152 | @@ -755,49 +772,59 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque) |
f00a720d TL |
153 | if (bitmap == NULL) { |
154 | bitmap = bdrv_create_dirty_bitmap(di->bs, dump_cb_block_size, PBS_BITMAP_NAME, task->errp); | |
155 | if (!bitmap) { | |
156 | - goto err; | |
157 | + goto err_mutex; | |
158 | } | |
159 | + action = PBS_BITMAP_ACTION_NEW; | |
160 | } else { | |
161 | expect_only_dirty = proxmox_backup_check_incremental(pbs, devname, di->size) != 0; | |
162 | } | |
163 | ||
164 | if (expect_only_dirty) { | |
165 | - dirty += bdrv_get_dirty_count(bitmap); | |
166 | + /* track clean chunks as reused */ | |
167 | + dirty = MIN(bdrv_get_dirty_count(bitmap), di->size); | |
168 | + backup_state.stat.reused += di->size - dirty; | |
169 | + action = PBS_BITMAP_ACTION_USED; | |
170 | } else { | |
171 | /* mark entire bitmap as dirty to make full backup */ | |
172 | bdrv_set_dirty_bitmap(bitmap, 0, di->size); | |
173 | - dirty += di->size; | |
174 | + if (action != PBS_BITMAP_ACTION_NEW) { | |
175 | + action = PBS_BITMAP_ACTION_INVALID; | |
176 | + } | |
177 | } | |
178 | di->bitmap = bitmap; | |
179 | } else { | |
180 | - dirty += di->size; | |
181 | - | |
182 | /* after a full backup the old dirty bitmap is invalid anyway */ | |
183 | if (bitmap != NULL) { | |
184 | bdrv_release_dirty_bitmap(bitmap); | |
185 | + action = PBS_BITMAP_ACTION_NOT_USED_REMOVED; | |
186 | } | |
187 | } | |
188 | ||
189 | int dev_id = proxmox_backup_co_register_image(pbs, devname, di->size, expect_only_dirty, task->errp); | |
190 | if (dev_id < 0) { | |
191 | - goto err; | |
192 | + goto err_mutex; | |
193 | } | |
194 | ||
195 | if (!(di->target = bdrv_backup_dump_create(dump_cb_block_size, di->size, pvebackup_co_dump_pbs_cb, di, task->errp))) { | |
196 | - goto err; | |
197 | + goto err_mutex; | |
198 | } | |
199 | ||
200 | di->dev_id = dev_id; | |
201 | + | |
202 | + PBSBitmapInfo *info = g_malloc(sizeof(*info)); | |
203 | + info->drive = g_strdup(devname); | |
204 | + info->action = action; | |
205 | + info->size = di->size; | |
206 | + info->dirty = dirty; | |
207 | + backup_state.stat.bitmap_list = g_list_append(backup_state.stat.bitmap_list, info); | |
208 | } | |
209 | } else if (format == BACKUP_FORMAT_VMA) { | |
210 | - dirty = total; | |
211 | - | |
212 | vmaw = vma_writer_create(task->backup_file, uuid, &local_err); | |
213 | if (!vmaw) { | |
214 | if (local_err) { | |
215 | error_propagate(task->errp, local_err); | |
216 | } | |
217 | - goto err; | |
218 | + goto err_mutex; | |
219 | } | |
220 | ||
221 | /* register all devices for vma writer */ | |
8dca018b | 222 | @@ -807,7 +834,7 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque) |
f00a720d TL |
223 | l = g_list_next(l); |
224 | ||
225 | if (!(di->target = bdrv_backup_dump_create(VMA_CLUSTER_SIZE, di->size, pvebackup_co_dump_vma_cb, di, task->errp))) { | |
226 | - goto err; | |
227 | + goto err_mutex; | |
228 | } | |
229 | ||
230 | const char *devname = bdrv_get_device_name(di->bs); | |
8dca018b | 231 | @@ -815,16 +842,14 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque) |
f00a720d TL |
232 | if (di->dev_id <= 0) { |
233 | error_set(task->errp, ERROR_CLASS_GENERIC_ERROR, | |
234 | "register_stream failed"); | |
235 | - goto err; | |
236 | + goto err_mutex; | |
237 | } | |
238 | } | |
239 | } else if (format == BACKUP_FORMAT_DIR) { | |
240 | - dirty = total; | |
241 | - | |
242 | if (mkdir(task->backup_file, 0640) != 0) { | |
243 | error_setg_errno(task->errp, errno, "can't create directory '%s'\n", | |
244 | task->backup_file); | |
245 | - goto err; | |
246 | + goto err_mutex; | |
247 | } | |
248 | backup_dir = task->backup_file; | |
249 | ||
8dca018b | 250 | @@ -841,18 +866,18 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque) |
f00a720d TL |
251 | di->size, flags, false, &local_err); |
252 | if (local_err) { | |
253 | error_propagate(task->errp, local_err); | |
254 | - goto err; | |
255 | + goto err_mutex; | |
256 | } | |
257 | ||
258 | di->target = bdrv_open(di->targetfile, NULL, NULL, flags, &local_err); | |
259 | if (!di->target) { | |
260 | error_propagate(task->errp, local_err); | |
261 | - goto err; | |
262 | + goto err_mutex; | |
263 | } | |
264 | } | |
265 | } else { | |
266 | error_set(task->errp, ERROR_CLASS_GENERIC_ERROR, "unknown backup format"); | |
267 | - goto err; | |
268 | + goto err_mutex; | |
269 | } | |
270 | ||
271 | ||
8dca018b | 272 | @@ -860,7 +885,7 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque) |
f00a720d TL |
273 | if (task->has_config_file) { |
274 | if (pvebackup_co_add_config(task->config_file, config_name, format, backup_dir, | |
275 | vmaw, pbs, task->errp) != 0) { | |
276 | - goto err; | |
277 | + goto err_mutex; | |
278 | } | |
279 | } | |
280 | ||
8dca018b | 281 | @@ -868,12 +893,11 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque) |
f00a720d TL |
282 | if (task->has_firewall_file) { |
283 | if (pvebackup_co_add_config(task->firewall_file, firewall_name, format, backup_dir, | |
284 | vmaw, pbs, task->errp) != 0) { | |
285 | - goto err; | |
286 | + goto err_mutex; | |
287 | } | |
288 | } | |
289 | /* initialize global backup_state now */ | |
290 | - | |
291 | - qemu_mutex_lock(&backup_state.stat.lock); | |
292 | + /* note: 'reused' and 'bitmap_list' are initialized earlier */ | |
293 | ||
294 | if (backup_state.stat.error) { | |
295 | error_free(backup_state.stat.error); | |
8dca018b | 296 | @@ -893,10 +917,9 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque) |
f00a720d TL |
297 | char *uuid_str = g_strdup(backup_state.stat.uuid_str); |
298 | ||
299 | backup_state.stat.total = total; | |
300 | - backup_state.stat.dirty = dirty; | |
301 | + backup_state.stat.dirty = total - backup_state.stat.reused; | |
302 | backup_state.stat.transferred = 0; | |
303 | backup_state.stat.zero_bytes = 0; | |
304 | - backup_state.stat.reused = format == BACKUP_FORMAT_PBS && dirty >= total ? 0 : total - dirty; | |
305 | ||
306 | qemu_mutex_unlock(&backup_state.stat.lock); | |
307 | ||
8dca018b | 308 | @@ -913,6 +936,9 @@ static void coroutine_fn pvebackup_co_prepare(void *opaque) |
f00a720d TL |
309 | task->result = uuid_info; |
310 | return; | |
311 | ||
312 | +err_mutex: | |
313 | + qemu_mutex_unlock(&backup_state.stat.lock); | |
314 | + | |
315 | err: | |
316 | ||
317 | l = di_list; | |
8dca018b | 318 | @@ -1076,11 +1102,42 @@ BackupStatus *qmp_query_backup(Error **errp) |
f00a720d TL |
319 | return info; |
320 | } | |
321 | ||
322 | +PBSBitmapInfoList *qmp_query_pbs_bitmap_info(Error **errp) | |
323 | +{ | |
324 | + PBSBitmapInfoList *head = NULL, **p_next = &head; | |
325 | + | |
326 | + qemu_mutex_lock(&backup_state.stat.lock); | |
327 | + | |
328 | + GList *l = backup_state.stat.bitmap_list; | |
329 | + while (l) { | |
330 | + PBSBitmapInfo *info = (PBSBitmapInfo *)l->data; | |
331 | + l = g_list_next(l); | |
332 | + | |
333 | + /* clone bitmap info to avoid auto free after QMP marshalling */ | |
334 | + PBSBitmapInfo *info_ret = g_malloc0(sizeof(*info_ret)); | |
335 | + info_ret->drive = g_strdup(info->drive); | |
336 | + info_ret->action = info->action; | |
337 | + info_ret->size = info->size; | |
338 | + info_ret->dirty = info->dirty; | |
339 | + | |
340 | + PBSBitmapInfoList *info_list = g_malloc0(sizeof(*info_list)); | |
341 | + info_list->value = info_ret; | |
342 | + | |
343 | + *p_next = info_list; | |
344 | + p_next = &info_list->next; | |
345 | + } | |
346 | + | |
347 | + qemu_mutex_unlock(&backup_state.stat.lock); | |
348 | + | |
349 | + return head; | |
350 | +} | |
351 | + | |
352 | ProxmoxSupportStatus *qmp_query_proxmox_support(Error **errp) | |
353 | { | |
354 | ProxmoxSupportStatus *ret = g_malloc0(sizeof(*ret)); | |
32ee4115 | 355 | ret->pbs_library_version = g_strdup(proxmox_backup_qemu_version()); |
f00a720d | 356 | ret->pbs_dirty_bitmap = true; |
e9b36665 | 357 | ret->pbs_dirty_bitmap_savevm = true; |
f00a720d TL |
358 | + ret->query_bitmap_info = true; |
359 | return ret; | |
360 | } | |
361 | diff --git a/qapi/block-core.json b/qapi/block-core.json | |
5b15e2ec | 362 | index 0b453c61d4..16e184dd28 100644 |
f00a720d TL |
363 | --- a/qapi/block-core.json |
364 | +++ b/qapi/block-core.json | |
5b15e2ec | 365 | @@ -871,6 +871,8 @@ |
f00a720d TL |
366 | # @pbs-dirty-bitmap: True if dirty-bitmap-incremental backups to PBS are |
367 | # supported. | |
368 | # | |
369 | +# @query-bitmap-info: True if the 'query-pbs-bitmap-info' QMP call is supported. | |
370 | +# | |
e9b36665 SR |
371 | # @pbs-dirty-bitmap-savevm: True if 'dirty-bitmaps' migration capability can |
372 | # safely be set for savevm-async. | |
32ee4115 | 373 | # |
5b15e2ec | 374 | @@ -879,6 +881,7 @@ |
f00a720d TL |
375 | ## |
376 | { 'struct': 'ProxmoxSupportStatus', | |
32ee4115 SR |
377 | 'data': { 'pbs-dirty-bitmap': 'bool', |
378 | + 'query-bitmap-info': 'bool', | |
e9b36665 | 379 | 'pbs-dirty-bitmap-savevm': 'bool', |
32ee4115 | 380 | 'pbs-library-version': 'str' } } |
f00a720d | 381 | |
5b15e2ec | 382 | @@ -892,6 +895,59 @@ |
f00a720d TL |
383 | ## |
384 | { 'command': 'query-proxmox-support', 'returns': 'ProxmoxSupportStatus' } | |
385 | ||
386 | +## | |
387 | +# @PBSBitmapAction: | |
388 | +# | |
389 | +# An action taken on a dirty-bitmap when a backup job was started. | |
390 | +# | |
391 | +# @not-used: Bitmap mode was not enabled. | |
392 | +# | |
393 | +# @not-used-removed: Bitmap mode was not enabled, but a bitmap from a | |
394 | +# previous backup still existed and was removed. | |
395 | +# | |
396 | +# @new: A new bitmap was attached to the drive for this backup. | |
397 | +# | |
398 | +# @used: An existing bitmap will be used to only backup changed data. | |
399 | +# | |
400 | +# @invalid: A bitmap existed, but had to be cleared since it's associated | |
401 | +# base snapshot did not match the base given for the current job or | |
402 | +# the crypt mode has changed. | |
403 | +# | |
404 | +## | |
405 | +{ 'enum': 'PBSBitmapAction', | |
406 | + 'data': ['not-used', 'not-used-removed', 'new', 'used', 'invalid'] } | |
407 | + | |
408 | +## | |
409 | +# @PBSBitmapInfo: | |
410 | +# | |
411 | +# Contains information about dirty bitmaps used for each drive in a PBS backup. | |
412 | +# | |
413 | +# @drive: The underlying drive. | |
414 | +# | |
415 | +# @action: The action that was taken when the backup started. | |
416 | +# | |
417 | +# @size: The total size of the drive. | |
418 | +# | |
419 | +# @dirty: How much of the drive is considered dirty and will be backed up, | |
420 | +# or 'size' if everything will be. | |
421 | +# | |
422 | +## | |
423 | +{ 'struct': 'PBSBitmapInfo', | |
424 | + 'data': { 'drive': 'str', 'action': 'PBSBitmapAction', 'size': 'int', | |
425 | + 'dirty': 'int' } } | |
426 | + | |
427 | +## | |
428 | +# @query-pbs-bitmap-info: | |
429 | +# | |
430 | +# Returns information about dirty bitmaps used on the most recently started | |
431 | +# backup. Returns nothing when the last backup was not using PBS or if no | |
432 | +# backup occured in this session. | |
433 | +# | |
434 | +# Returns: @PBSBitmapInfo | |
435 | +# | |
436 | +## | |
437 | +{ 'command': 'query-pbs-bitmap-info', 'returns': ['PBSBitmapInfo'] } | |
438 | + | |
439 | ## | |
440 | # @BlockDeviceTimedStats: | |
441 | # |