From 89af8a77eeddf3516837f76687b0b845fbb6c0b6 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Wed, 20 Feb 2013 10:36:15 +0100 Subject: [PATCH] update to latest backup patches --- debian/patches/0000-cover-letter.patch | 88 +++++++++++++++ ...cumenation-for-new-backup-framework.patch} | 50 ++------- ...basic-backup-support-to-block-driver.patch | 32 +++--- ...-add-backup-related-monitor-commands.patch | 100 +++++++++--------- ...004-introduce-new-vma-archive-format.patch | 32 +++--- ...0005-add-regression-tests-for-backup.patch | 22 ++-- .../0006-add-vm-state-to-backups.patch | 53 +++++----- debian/patches/series | 4 +- 8 files changed, 223 insertions(+), 158 deletions(-) create mode 100644 debian/patches/0000-cover-letter.patch rename debian/patches/{0001-RFC-Efficient-VM-backup-for-qemu.patch => 0001-add-documenation-for-new-backup-framework.patch} (75%) diff --git a/debian/patches/0000-cover-letter.patch b/debian/patches/0000-cover-letter.patch new file mode 100644 index 0000000..ec50591 --- /dev/null +++ b/debian/patches/0000-cover-letter.patch @@ -0,0 +1,88 @@ +From de76a5c6306d782733217668f3f55c92aa2be36a Mon Sep 17 00:00:00 2001 +From: Dietmar Maurer +Date: Wed, 20 Feb 2013 10:26:23 +0100 +Subject: [PATCH v4 0/6] Efficient VM backup for qemu + +This series provides a way to efficiently backup VMs. + +* Backup to a single archive file +* Backup contain all data to restore VM (full backup) +* Do not depend on storage type or image format +* Avoid use of temporary storage +* store sparse images efficiently + +The file docs/backup.txt contains more details. + +Changes since v1: + +* fix spelling errors +* move BackupInfo from BDS to BackupBlockJob +* introduce BackupDriver to allow more than one backup format +* vma: add suport to store vmstate (size is not known in advance) +* add ability to store VM state + +Changes since v2: + +* BackupDriver: remove cancel_cb +* use enum for BackupFormat +* vma: use bdrv_open instead of bdrv_file_open +* vma: fix aio, use O_DIRECT +* backup one drive after another (try to avoid high load) + +Changes since v3: + +* move reviewer info from commit messages to cover-letter +* remove 'RFC' from log headers and file names +* removed comment about slow qcow2 snapshot bug +* fix spelling errors +* fixed copyright +* change 'backupfile' parameter name to 'backup-file' +* change 'config-filename' parameter name to 'config-file' +* add documentation for 'devlist' parameter. +* rename backup_cancel to backup-cancel + + +Dietmar Maurer (6): + add documenation for new backup framework + add basic backup support to block driver + add backup related monitor commands + introduce new vma archive format + add regression tests for backup + add vm state to backups + + Makefile | 3 +- + Makefile.objs | 1 + + backup.c | 338 +++++++++++++++++ + backup.h | 45 +++ + block.c | 71 ++++- + blockdev.c | 617 ++++++++++++++++++++++++++++++ + docs/backup.txt | 116 ++++++ + docs/specs/vma_spec.txt | 24 ++ + hmp-commands.hx | 31 ++ + hmp.c | 64 ++++ + hmp.h | 3 + + include/block/block.h | 2 + + include/block/blockjob.h | 10 + + monitor.c | 7 + + qapi-schema.json | 98 +++++ + qmp-commands.hx | 27 ++ + tests/Makefile | 11 +- + tests/backup-test.c | 517 +++++++++++++++++++++++++ + vma-reader.c | 799 +++++++++++++++++++++++++++++++++++++++ + vma-writer.c | 932 ++++++++++++++++++++++++++++++++++++++++++++++ + vma.c | 559 +++++++++++++++++++++++++++ + vma.h | 145 +++++++ + 22 files changed, 4411 insertions(+), 9 deletions(-) + create mode 100644 backup.c + create mode 100644 backup.h + create mode 100644 docs/backup.txt + create mode 100644 docs/specs/vma_spec.txt + create mode 100644 tests/backup-test.c + create mode 100644 vma-reader.c + create mode 100644 vma-writer.c + create mode 100644 vma.c + create mode 100644 vma.h + +-- +1.7.2.5 + diff --git a/debian/patches/0001-RFC-Efficient-VM-backup-for-qemu.patch b/debian/patches/0001-add-documenation-for-new-backup-framework.patch similarity index 75% rename from debian/patches/0001-RFC-Efficient-VM-backup-for-qemu.patch rename to debian/patches/0001-add-documenation-for-new-backup-framework.patch index 09df992..a811415 100644 --- a/debian/patches/0001-RFC-Efficient-VM-backup-for-qemu.patch +++ b/debian/patches/0001-add-documenation-for-new-backup-framework.patch @@ -1,47 +1,22 @@ -From e9d7fc0567eb5bef368a950a4813d6eb5867572b Mon Sep 17 00:00:00 2001 +From 2f0dcd89a0de8b656d33ce6997c09879bd287af7 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Tue, 13 Nov 2012 09:24:50 +0100 -Subject: [PATCH v3 1/6] RFC: Efficient VM backup for qemu +Subject: [PATCH v4 1/6] add documenation for new backup framework -This series provides a way to efficiently backup VMs. - -* Backup to a single archive file -* Backup contain all data to restore VM (full backup) -* Do not depend on storage type or image format -* Avoid use of temporary storage -* store sparse images efficiently - -The file docs/backup-rfc.txt contains more details. - -Changes since v1: - -* fix spelling errors -* move BackupInfo from BDS to BackupBlockJob -* introduce BackupDriver to allow more than one backup format -* vma: add suport to store vmstate (size is not known in advance) -* add ability to store VM state - -Changes since v2: - -* BackupDriver: remove cancel_cb -* use enum for BackupFormat -* vma: use bdrv_open instead of bdrv_file_open -* vma: fix aio, use O_DIRECT -* backup one drive after another (try to avoid high load) Signed-off-by: Dietmar Maurer --- - docs/backup-rfc.txt | 119 +++++++++++++++++++++++++++++++++++++++++++++++++++ - 1 files changed, 119 insertions(+), 0 deletions(-) - create mode 100644 docs/backup-rfc.txt + docs/backup.txt | 116 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ + 1 files changed, 116 insertions(+), 0 deletions(-) + create mode 100644 docs/backup.txt -diff --git a/docs/backup-rfc.txt b/docs/backup-rfc.txt +diff --git a/docs/backup.txt b/docs/backup.txt new file mode 100644 -index 0000000..5b4b3df +index 0000000..927d787 --- /dev/null -+++ b/docs/backup-rfc.txt -@@ -0,0 +1,119 @@ -+RFC: Efficient VM backup for qemu ++++ b/docs/backup.txt +@@ -0,0 +1,116 @@ ++Efficient VM backup for qemu + +=Requirements= + @@ -88,9 +63,6 @@ index 0000000..5b4b3df +of reference counting (rados, sheepdog, dm-thin, qcow2). It would be possible +to use that for backups, but for now we want to be storage-independent. + -+Note: It turned out that taking a qcow2 snapshot can take a very long -+time on larger files. -+ +=Make it more efficient= + +The be more efficient, we simply need to avoid unnecessary steps. The @@ -100,7 +72,7 @@ index 0000000..5b4b3df +2.) write that data into the backup archive +3.) write new data (VM write) + -+As you can see, this involves only one read, an two writes. ++As you can see, this involves only one read, and two writes. + +To make that work, our backup archive need to be able to store image +data 'out of order'. It is important to notice that this will not work diff --git a/debian/patches/0002-add-basic-backup-support-to-block-driver.patch b/debian/patches/0002-add-basic-backup-support-to-block-driver.patch index a43bb03..8714e9f 100644 --- a/debian/patches/0002-add-basic-backup-support-to-block-driver.patch +++ b/debian/patches/0002-add-basic-backup-support-to-block-driver.patch @@ -1,7 +1,7 @@ -From eb29d6acb8d8e671e292c9f50655bc87d9742282 Mon Sep 17 00:00:00 2001 +From 940afda26b17f3d5776e4809e6dfce5cee44c102 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Tue, 13 Nov 2012 10:03:52 +0100 -Subject: [PATCH v3 2/6] add basic backup support to block driver +Subject: [PATCH v4 2/6] add basic backup support to block driver Function backup_job_create() creates a block job to backup a block device. The coroutine is started with backup_job_start(). @@ -26,7 +26,7 @@ Signed-off-by: Dietmar Maurer create mode 100644 backup.h diff --git a/Makefile.objs b/Makefile.objs -index 21e9c91..d72822b 100644 +index a68cdac..df64f70 100644 --- a/Makefile.objs +++ b/Makefile.objs @@ -13,6 +13,7 @@ block-obj-$(CONFIG_POSIX) += aio-posix.o @@ -39,14 +39,14 @@ index 21e9c91..d72822b 100644 block-obj-y += qemu-coroutine-sleep.o diff --git a/backup.c b/backup.c new file mode 100644 -index 0000000..61ca668 +index 0000000..c9576d5 --- /dev/null +++ b/backup.c @@ -0,0 +1,338 @@ +/* + * QEMU backup + * -+ * Copyright (C) 2012 Proxmox Server Solutions ++ * Copyright (C) 2013 Proxmox Server Solutions + * + * Authors: + * Dietmar Maurer (dietmar@proxmox.com) @@ -219,7 +219,7 @@ index 0000000..61ca668 +static void backup_set_speed(BlockJob *job, int64_t speed, Error **errp) +{ + BackupBlockJob *s = container_of(job, BackupBlockJob, common); -+ ++ + if (speed < 0) { + error_set(errp, QERR_INVALID_PARAMETER, "speed"); + return; @@ -260,17 +260,17 @@ index 0000000..61ca668 + + /* we need to yield so that qemu_aio_flush() returns. + * (without, VM does not reboot) -+ * Note: use 1000 instead of 0 (0 priorize this task too much) ++ * Note: use 1000 instead of 0 (0 prioritize this task too much) + */ -+ if (job->common.speed) { ++ if (job->common.speed) { + uint64_t delay_ns = ratelimit_calculate_delay( + &job->limit, job->sectors_read); + job->sectors_read = 0; + block_job_sleep_ns(&job->common, rt_clock, delay_ns); -+ } else { ++ } else { + block_job_sleep_ns(&job->common, rt_clock, 1000); -+ } -+ ++ } ++ + if (block_job_is_cancelled(&job->common)) { + ret = -1; + break; @@ -378,19 +378,19 @@ index 0000000..61ca668 + job->bitmap = g_new0(unsigned long, bitmap_size); + + job->common.len = bs->total_sectors*BDRV_SECTOR_SIZE; -+ ++ + return 0; +} diff --git a/backup.h b/backup.h new file mode 100644 -index 0000000..20a9016 +index 0000000..d9395bc --- /dev/null +++ b/backup.h @@ -0,0 +1,32 @@ +/* + * QEMU backup related definitions + * -+ * Copyright (C) Proxmox Server Solutions ++ * Copyright (C) 2013 Proxmox Server Solutions + * + * Authors: + * Dietmar Maurer (dietmar@proxmox.com) @@ -415,8 +415,8 @@ index 0000000..20a9016 +void backup_job_start(BlockDriverState *bs, bool cancel); + +int backup_job_create(BlockDriverState *bs, BackupDumpFunc *backup_dump_cb, -+ BlockDriverCompletionFunc *backup_complete_cb, -+ void *opaque, int64_t speed); ++ BlockDriverCompletionFunc *backup_complete_cb, ++ void *opaque, int64_t speed); + +#endif /* QEMU_BACKUP_H */ diff --git a/block.c b/block.c diff --git a/debian/patches/0003-add-backup-related-monitor-commands.patch b/debian/patches/0003-add-backup-related-monitor-commands.patch index a9135f7..ea7a2e4 100644 --- a/debian/patches/0003-add-backup-related-monitor-commands.patch +++ b/debian/patches/0003-add-backup-related-monitor-commands.patch @@ -1,7 +1,7 @@ -From 02f8d395026b507cdcaaec00b324a67779298383 Mon Sep 17 00:00:00 2001 +From 982a8ac63f778110ad89b4dc415011166c9dcd8e Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Tue, 13 Nov 2012 11:27:56 +0100 -Subject: [PATCH v3 3/6] add backup related monitor commands +Subject: [PATCH v4 3/6] add backup related monitor commands We use a generic BackupDriver struct to encapsulate all archive format related function. @@ -18,17 +18,17 @@ Signed-off-by: Dietmar Maurer hmp.c | 63 ++++++++ hmp.h | 3 + monitor.c | 7 + - qapi-schema.json | 91 ++++++++++++ + qapi-schema.json | 95 ++++++++++++ qmp-commands.hx | 27 ++++ - 8 files changed, 657 insertions(+), 0 deletions(-) + 8 files changed, 661 insertions(+), 0 deletions(-) diff --git a/backup.h b/backup.h -index 20a9016..be52ea4 100644 +index d9395bc..c8ba153 100644 --- a/backup.h +++ b/backup.h @@ -29,4 +29,16 @@ int backup_job_create(BlockDriverState *bs, BackupDumpFunc *backup_dump_cb, - BlockDriverCompletionFunc *backup_complete_cb, - void *opaque, int64_t speed); + BlockDriverCompletionFunc *backup_complete_cb, + void *opaque, int64_t speed); +typedef struct BackupDriver { + const char *format; @@ -44,7 +44,7 @@ index 20a9016..be52ea4 100644 + #endif /* QEMU_BACKUP_H */ diff --git a/blockdev.c b/blockdev.c -index 63e6f1e..92a7ca3 100644 +index 63e6f1e..c340fde 100644 --- a/blockdev.c +++ b/blockdev.c @@ -20,6 +20,7 @@ @@ -71,7 +71,7 @@ index 63e6f1e..92a7ca3 100644 + int64_t speed; + time_t start_time; + time_t end_time; -+ char *backupfile; ++ char *backup_file; + const BackupDriver *driver; + void *writer; + GList *bcb_list; @@ -173,12 +173,12 @@ index 63e6f1e..92a7ca3 100644 + while (l) { + BackupCB *bcb = l->data; + l = g_list_next(l); -+ BlockJob *job = bcb->bs->job; -+ if (job) { ++ BlockJob *job = bcb->bs->job; ++ if (job) { + job_count++; + if (!bcb->started) { -+ bcb->started = true; -+ backup_job_start(bcb->bs, true); ++ bcb->started = true; ++ backup_job_start(bcb->bs, true); + } + if (!bcb->completed) { + block_job_cancel_sync(job); @@ -197,7 +197,7 @@ index 63e6f1e..92a7ca3 100644 +static void backup_run_next_job(void) +{ + GList *l = backup_state.bcb_list; -+ while(l) { ++ while (l) { + BackupCB *bcb = l->data; + l = g_list_next(l); + @@ -205,7 +205,7 @@ index 63e6f1e..92a7ca3 100644 + if (!bcb->started) { + bcb->started = true; + bool cancel = backup_state.error || backup_state.cancel; -+ backup_job_start(bcb->bs,cancel); ++ backup_job_start(bcb->bs, cancel); + } + return; + } @@ -233,8 +233,8 @@ index 63e6f1e..92a7ca3 100644 + backup_run_next_job(); +} + -+char *qmp_backup(const char *backupfile, bool has_format, BackupFormat format, -+ bool has_config_filename, const char *config_filename, ++char *qmp_backup(const char *backup_file, bool has_format, BackupFormat format, ++ bool has_config_file, const char *config_file, + bool has_devlist, const char *devlist, + bool has_speed, int64_t speed, Error **errp) +{ @@ -319,7 +319,7 @@ index 63e6f1e..92a7ca3 100644 + + uuid_generate(uuid); + -+ writer = driver->open_cb(backupfile, uuid, &local_err); ++ writer = driver->open_cb(backup_file, uuid, &local_err); + if (!writer) { + if (error_is_set(&local_err)) { + error_propagate(errp, local_err); @@ -348,16 +348,16 @@ index 63e6f1e..92a7ca3 100644 + } + + /* add configuration file to archive */ -+ if (has_config_filename) { ++ if (has_config_file) { + char *cdata = NULL; + gsize clen = 0; + GError *err = NULL; -+ if (!g_file_get_contents(config_filename, &cdata, &clen, &err)) { -+ error_setg(errp, "unable to read file '%s'", config_filename); ++ if (!g_file_get_contents(config_file, &cdata, &clen, &err)) { ++ error_setg(errp, "unable to read file '%s'", config_file); + goto err; + } + -+ const char *basename = g_path_get_basename(config_filename); ++ const char *basename = g_path_get_basename(config_file); + if (driver->register_config_cb(writer, basename, cdata, clen) < 0) { + error_setg(errp, "register_config failed"); + g_free(cdata); @@ -382,10 +382,10 @@ index 63e6f1e..92a7ca3 100644 + backup_state.start_time = time(NULL); + backup_state.end_time = 0; + -+ if (backup_state.backupfile) { -+ g_free(backup_state.backupfile); ++ if (backup_state.backup_file) { ++ g_free(backup_state.backup_file); + } -+ backup_state.backupfile = g_strdup(backupfile); ++ backup_state.backup_file = g_strdup(backup_file); + + backup_state.writer = writer; + @@ -426,7 +426,7 @@ index 63e6f1e..92a7ca3 100644 + } + + if (writer) { -+ unlink(backupfile); ++ unlink(backup_file); + if (driver) { + Error *err = NULL; + driver->close_cb(writer, &err); @@ -449,9 +449,9 @@ index 63e6f1e..92a7ca3 100644 + info->has_start_time = true; + info->start_time = backup_state.start_time; + -+ if (backup_state.backupfile) { -+ info->has_backupfile = true; -+ info->backupfile = g_strdup(backup_state.backupfile); ++ if (backup_state.backup_file) { ++ info->has_backup_file = true; ++ info->backup_file = g_strdup(backup_state.backup_file); + } + + info->has_uuid = true; @@ -534,7 +534,7 @@ index 64008a9..0f178d8 100644 show current migration capabilities @item info migrate_cache_size diff --git a/hmp.c b/hmp.c -index 2f47a8a..9ac34c5 100644 +index 2f47a8a..b2c1f23 100644 --- a/hmp.c +++ b/hmp.c @@ -131,6 +131,38 @@ void hmp_info_mice(Monitor *mon, const QDict *qdict) @@ -554,14 +554,14 @@ index 2f47a8a..9ac34c5 100644 + monitor_printf(mon, "Backup status: %s\n", info->status); + } + } -+ if (info->has_backupfile) { ++ if (info->has_backup_file) { + int per = (info->has_total && info->total && + info->has_transferred && info->transferred) ? + (info->transferred * 100)/info->total : 0; + int zero_per = (info->has_total && info->total && + info->has_zero_bytes && info->zero_bytes) ? + (info->zero_bytes * 100)/info->total : 0; -+ monitor_printf(mon, "Backup file: %s\n", info->backupfile); ++ monitor_printf(mon, "Backup file: %s\n", info->backup_file); + monitor_printf(mon, "Backup uuid: %s\n", info->uuid); + monitor_printf(mon, "Total size: %zd\n", info->total); + monitor_printf(mon, "Transferred bytes: %zd (%d%%)\n", @@ -595,13 +595,13 @@ index 2f47a8a..9ac34c5 100644 + +void hmp_backup(Monitor *mon, const QDict *qdict) +{ -+ const char *backupfile = qdict_get_str(qdict, "backupfile"); ++ const char *backup_file = qdict_get_str(qdict, "backup-file"); + const char *devlist = qdict_get_try_str(qdict, "devlist"); + int64_t speed = qdict_get_try_int(qdict, "speed", 0); + + Error *errp = NULL; + -+ qmp_backup(backupfile, true, BACKUP_FORMAT_VMA, false, NULL, !!devlist, ++ qmp_backup(backup_file, true, BACKUP_FORMAT_VMA, false, NULL, !!devlist, + devlist, qdict_haskey(qdict, "speed"), speed, &errp); + + if (error_is_set(&errp)) { @@ -636,7 +636,7 @@ index 30b3c20..ad4cf80 100644 void hmp_block_job_cancel(Monitor *mon, const QDict *qdict); void hmp_block_job_pause(Monitor *mon, const QDict *qdict); diff --git a/monitor.c b/monitor.c -index 20bd19b..5f979ce 100644 +index 6a0f257..e4a810c 100644 --- a/monitor.c +++ b/monitor.c @@ -2666,6 +2666,13 @@ static mon_cmd_t info_cmds[] = { @@ -654,7 +654,7 @@ index 20bd19b..5f979ce 100644 .args_type = "", .params = "", diff --git a/qapi-schema.json b/qapi-schema.json -index bd289ae..c91df47 100644 +index 7275b5d..09ca8ef 100644 --- a/qapi-schema.json +++ b/qapi-schema.json @@ -425,6 +425,39 @@ @@ -685,19 +685,19 @@ index bd289ae..c91df47 100644 +# +# @uuid: #optional uuid for this backup job +# -+# Since: 1.4.0 ++# Since: 1.5.0 +## +{ 'type': 'BackupStatus', + 'data': {'*status': 'str', '*errmsg': 'str', '*total': 'int', + '*transferred': 'int', '*zero-bytes': 'int', + '*start-time': 'int', '*end-time': 'int', -+ '*backupfile': 'str', '*uuid': 'str' } } ++ '*backup-file': 'str', '*uuid': 'str' } } + +## # @query-events: # # Return a list of supported QMP events by this server -@@ -1824,6 +1857,64 @@ +@@ -1824,6 +1857,68 @@ 'data': { 'path': 'str' }, 'returns': [ 'ObjectPropertyInfo' ] } @@ -717,7 +717,7 @@ index bd289ae..c91df47 100644 +# +# Starts a VM backup. +# -+# @backupfile: the backup file name ++# @backup-file: the backup file name +# +# @format: format of the backup file +# @@ -726,12 +726,16 @@ index bd289ae..c91df47 100644 +# +# @speed: #optional the maximum speed, in bytes per second +# ++# @devlist: #optional list of block device names (separated by ',', ';' ++# or ':'). By default the backup includes all writable block devices. ++# +# Returns: the uuid of the backup job +# -+# Since: 1.4.0 ++# Since: 1.5.0 +## -+{ 'command': 'backup', 'data': { 'backupfile': 'str', '*format': 'BackupFormat', -+ '*config-filename': 'str', ++{ 'command': 'backup', 'data': { 'backup-file': 'str', ++ '*format': 'BackupFormat', ++ '*config-file': 'str', + '*devlist': 'str', '*speed': 'int' }, + 'returns': 'str' } + @@ -742,7 +746,7 @@ index bd289ae..c91df47 100644 +# +# Returns: @BackupStatus +# -+# Since: 1.4.0 ++# Since: 1.5.0 +## +{ 'command': 'query-backup', 'returns': 'BackupStatus' } + @@ -755,7 +759,7 @@ index bd289ae..c91df47 100644 +# +# Notes: This command succeeds even if there is no backup process running. +# -+# Since: 1.4.0 ++# Since: 1.5.0 +## +{ 'command': 'backup-cancel' } + @@ -763,7 +767,7 @@ index bd289ae..c91df47 100644 # @qom-get: # diff --git a/qmp-commands.hx b/qmp-commands.hx -index 799adea..ca601b3 100644 +index 799adea..17e381b 100644 --- a/qmp-commands.hx +++ b/qmp-commands.hx @@ -889,6 +889,18 @@ EQMP @@ -771,12 +775,12 @@ index 799adea..ca601b3 100644 { + .name = "backup", -+ .args_type = "backupfile:s,format:s?,config-filename:F?,speed:o?,devlist:s?", ++ .args_type = "backup-file:s,format:s?,config-file:F?,speed:o?,devlist:s?", + .mhandler.cmd_new = qmp_marshal_input_backup, + }, + + { -+ .name = "backup_cancel", ++ .name = "backup-cancel", + .args_type = "", + .mhandler.cmd_new = qmp_marshal_input_backup_cancel, + }, diff --git a/debian/patches/0004-introduce-new-vma-archive-format.patch b/debian/patches/0004-introduce-new-vma-archive-format.patch index 0b54ae3..905c587 100644 --- a/debian/patches/0004-introduce-new-vma-archive-format.patch +++ b/debian/patches/0004-introduce-new-vma-archive-format.patch @@ -1,7 +1,7 @@ -From c04d17631e00ff368e42dc5d24e94152b6526014 Mon Sep 17 00:00:00 2001 +From 5476ae43806488e74cd293bbaa17f130aa53d402 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Tue, 13 Nov 2012 11:11:38 +0100 -Subject: [PATCH v3 4/6] introduce new vma archive format +Subject: [PATCH v4 4/6] introduce new vma archive format This is a very simple archive format, see docs/specs/vma_spec.txt @@ -45,7 +45,7 @@ index 0d9099a..16f1c25 100644 qemu-bridge-helper$(EXESUF): qemu-bridge-helper.o diff --git a/Makefile.objs b/Makefile.objs -index d72822b..111b9f8 100644 +index df64f70..91f133b 100644 --- a/Makefile.objs +++ b/Makefile.objs @@ -13,7 +13,7 @@ block-obj-$(CONFIG_POSIX) += aio-posix.o @@ -58,7 +58,7 @@ index d72822b..111b9f8 100644 block-obj-y += qemu-coroutine.o qemu-coroutine-lock.o qemu-coroutine-io.o block-obj-y += qemu-coroutine-sleep.o diff --git a/backup.h b/backup.h -index be52ea4..3a81e52 100644 +index c8ba153..406f011 100644 --- a/backup.h +++ b/backup.h @@ -15,6 +15,7 @@ @@ -70,7 +70,7 @@ index be52ea4..3a81e52 100644 #define BACKUP_CLUSTER_BITS 16 #define BACKUP_CLUSTER_SIZE (1<co_writer != 0); +} + @@ -1179,12 +1179,12 @@ index 0000000..04a346e + vmaw->co_writer = qemu_coroutine_self(); + + qemu_aio_set_fd_handler(vmaw->fd, NULL, vma_co_continue_write, -+ vma_co_write_finished, vmaw); ++ vma_co_write_finished, vmaw); + + DPRINTF("vma_co_write wait until writable\n"); + qemu_coroutine_yield(); + DPRINTF("vma_co_write starting %zd\n", bytes); -+ ++ + while (done < bytes) { + ret = write(vmaw->fd, buf + done, bytes - done); + if (ret > 0) { @@ -1870,7 +1870,7 @@ index 0000000..04a346e + diff --git a/vma.c b/vma.c new file mode 100644 -index 0000000..64da334 +index 0000000..b2e276c --- /dev/null +++ b/vma.c @@ -0,0 +1,559 @@ @@ -2159,10 +2159,10 @@ index 0000000..64da334 + dirname, di->devname); + printf("DEVINFO %s %zd\n", devfn, di->size); + -+ bdrv_img_create(devfn, "raw", NULL, NULL, NULL, di->size, ++ bdrv_img_create(devfn, "raw", NULL, NULL, NULL, di->size, + flags, &errp); + if (error_is_set(&errp)) { -+ g_error("can't create file %s: %s", devfn, ++ g_error("can't create file %s: %s", devfn, + error_get_pretty(errp)); + } + @@ -2337,12 +2337,12 @@ index 0000000..64da334 + bcb->vmaw = vmaw; + bcb->dev_id = dev_id; + -+ if (backup_job_create(bs, backup_dump_cb, backup_complete_cb, -+ bcb, 0) < 0) { ++ if (backup_job_create(bs, backup_dump_cb, backup_complete_cb, ++ bcb, 0) < 0) { + unlink(archivename); + g_error("backup_job_start failed"); + } else { -+ backup_job_start(bs, false); ++ backup_job_start(bs, false); + } + } + diff --git a/debian/patches/0005-add-regression-tests-for-backup.patch b/debian/patches/0005-add-regression-tests-for-backup.patch index 8c6dc11..dbf50f9 100644 --- a/debian/patches/0005-add-regression-tests-for-backup.patch +++ b/debian/patches/0005-add-regression-tests-for-backup.patch @@ -1,7 +1,7 @@ -From 6b0aa521526e0cc35539ee327b3e6c976da6e361 Mon Sep 17 00:00:00 2001 +From 4fb44b8d03b764db04e7751a14055cbb06a7791d Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Wed, 14 Nov 2012 09:57:04 +0100 -Subject: [PATCH v3 5/6] add regression tests for backup +Subject: [PATCH v4 5/6] add regression tests for backup Simple regression tests using vma-reader and vma-writer. @@ -15,10 +15,10 @@ Signed-off-by: Dietmar Maurer create mode 100644 tests/backup-test.c diff --git a/tests/Makefile b/tests/Makefile -index a2d62b8..c3d8b4a 100644 +index 567e36e..136be84 100644 --- a/tests/Makefile +++ b/tests/Makefile -@@ -57,6 +57,8 @@ gcov-files-test-cutils-y += util/cutils.c +@@ -59,6 +59,8 @@ gcov-files-test-mul64-y = util/host-utils.c check-block-$(CONFIG_POSIX) += tests/qemu-iotests-quick.sh @@ -27,7 +27,7 @@ index a2d62b8..c3d8b4a 100644 # All QTests for now are POSIX-only, but the dependencies are # really in libqtest, not in the testcases themselves. check-qtest-i386-y = tests/fdc-test$(EXESUF) -@@ -100,6 +102,7 @@ tests/test-coroutine$(EXESUF): tests/test-coroutine.o $(block-obj-y) libqemuutil +@@ -102,6 +104,7 @@ tests/test-coroutine$(EXESUF): tests/test-coroutine.o $(block-obj-y) libqemuutil tests/test-aio$(EXESUF): tests/test-aio.o $(block-obj-y) libqemuutil.a libqemustub.a tests/test-thread-pool$(EXESUF): tests/test-thread-pool.o $(block-obj-y) libqemuutil.a libqemustub.a tests/test-iov$(EXESUF): tests/test-iov.o libqemuutil.a @@ -35,7 +35,7 @@ index a2d62b8..c3d8b4a 100644 tests/test-hbitmap$(EXESUF): tests/test-hbitmap.o libqemuutil.a libqemustub.a tests/test-x86-cpuid$(EXESUF): tests/test-x86-cpuid.o tests/test-xbzrle$(EXESUF): tests/test-xbzrle.o xbzrle.o page_cache.o libqemuutil.a -@@ -209,10 +212,14 @@ check-tests/qemu-iotests-quick.sh: tests/qemu-iotests-quick.sh qemu-img$(EXESUF) +@@ -213,10 +216,14 @@ check-tests/qemu-iotests-quick.sh: tests/qemu-iotests-quick.sh qemu-img$(EXESUF) # Consolidated targets @@ -54,7 +54,7 @@ index a2d62b8..c3d8b4a 100644 -include $(wildcard tests/*.d) diff --git a/tests/backup-test.c b/tests/backup-test.c new file mode 100644 -index 0000000..5ee3b90 +index 0000000..5ff6f1d --- /dev/null +++ b/tests/backup-test.c @@ -0,0 +1,517 @@ @@ -306,7 +306,7 @@ index 0000000..5ee3b90 + int flags = BDRV_O_NATIVE_AIO|BDRV_O_RDWR|BDRV_O_CACHE_WB; + + bdrv_img_create(TEST_IMG_RESTORE_NAME, "raw", NULL, NULL, NULL, -+ size, flags, &errp); ++ size, flags, &errp); + if (error_is_set(&errp)) { + g_error("can't create file %s: %s", TEST_IMG_RESTORE_NAME, + error_get_pretty(errp)); @@ -433,11 +433,11 @@ index 0000000..5ee3b90 + bcb.vmaw = vmaw; + bcb.dev_id = vma_writer_register_stream(vmaw, bdrv_get_device_name(bs), + bdrv_getlength(bs)); -+ if (backup_job_create(bs, backup_dump_cb, backup_complete_cb, &bcb, -+ speed) < 0) { ++ if (backup_job_create(bs, backup_dump_cb, backup_complete_cb, &bcb, ++ speed) < 0) { + g_error("backup_job_create failed"); + } else { -+ backup_job_start(bs, false); ++ backup_job_start(bs, false); + } + + request_term = false; diff --git a/debian/patches/0006-add-vm-state-to-backups.patch b/debian/patches/0006-add-vm-state-to-backups.patch index 2ee9176..530412a 100644 --- a/debian/patches/0006-add-vm-state-to-backups.patch +++ b/debian/patches/0006-add-vm-state-to-backups.patch @@ -1,26 +1,26 @@ -From 2b1f27dec5e3f38d00b7905870337025bed8f366 Mon Sep 17 00:00:00 2001 +From de76a5c6306d782733217668f3f55c92aa2be36a Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Thu, 29 Nov 2012 10:46:49 +0100 -Subject: [PATCH v3 6/6] add vm state to backups +Subject: [PATCH v4 6/6] add vm state to backups Signed-off-by: Dietmar Maurer --- blockdev.c | 196 +++++++++++++++++++++++++++++++++++++++++++++++++++++- hmp.c | 3 +- - qapi-schema.json | 6 +- - 3 files changed, 200 insertions(+), 5 deletions(-) + qapi-schema.json | 5 +- + 3 files changed, 200 insertions(+), 4 deletions(-) diff --git a/blockdev.c b/blockdev.c -index aad6e0c..2805a1e 100644 +index 1cfc780..1b81824 100644 --- a/blockdev.c +++ b/blockdev.c @@ -22,6 +22,8 @@ #include "sysemu/arch_init.h" #include "backup.h" #include "vma.h" -+#include "qemu-file.h" -+#include "migration.h" ++#include "migration/qemu-file.h" ++#include "migration/migration.h" static QTAILQ_HEAD(drivelist, DriveInfo) drives = QTAILQ_HEAD_INITIALIZER(drives); @@ -198,8 +198,8 @@ index aad6e0c..2805a1e 100644 + goto out; +} + - char *qmp_backup(const char *backupfile, bool has_format, BackupFormat format, - bool has_config_filename, const char *config_filename, + char *qmp_backup(const char *backup_file, bool has_format, BackupFormat format, + bool has_config_file, const char *config_file, bool has_devlist, const char *devlist, - bool has_speed, int64_t speed, Error **errp) + bool has_speed, int64_t speed, @@ -207,7 +207,7 @@ index aad6e0c..2805a1e 100644 { BlockDriverState *bs; Error *local_err = NULL; -@@ -1528,6 +1694,8 @@ char *qmp_backup(const char *backupfile, bool has_format, BackupFormat format, +@@ -1528,6 +1694,8 @@ char *qmp_backup(const char *backup_file, bool has_format, BackupFormat format, return NULL; } @@ -216,7 +216,7 @@ index aad6e0c..2805a1e 100644 /* Todo: try to auto-detect format based on file name */ format = has_format ? format : BACKUP_FORMAT_VMA; -@@ -1608,6 +1776,22 @@ char *qmp_backup(const char *backupfile, bool has_format, BackupFormat format, +@@ -1608,6 +1776,22 @@ char *qmp_backup(const char *backup_file, bool has_format, BackupFormat format, size_t total = 0; /* register all devices for vma writer */ @@ -239,7 +239,7 @@ index aad6e0c..2805a1e 100644 l = bcblist; while (l) { BackupCB *bcb = l->data; -@@ -1675,6 +1859,9 @@ char *qmp_backup(const char *backupfile, bool has_format, BackupFormat format, +@@ -1675,6 +1859,9 @@ char *qmp_backup(const char *backup_file, bool has_format, BackupFormat format, backup_state.total = total; backup_state.transferred = 0; backup_state.zero_bytes = 0; @@ -249,7 +249,7 @@ index aad6e0c..2805a1e 100644 /* Grab a reference so hotplug does not delete the * BlockDriverState from underneath us. -@@ -1686,7 +1873,12 @@ char *qmp_backup(const char *backupfile, bool has_format, BackupFormat format, +@@ -1686,7 +1873,12 @@ char *qmp_backup(const char *backup_file, bool has_format, BackupFormat format, drive_get_ref(drive_get_by_blockdev(bcb->bs)); } @@ -258,19 +258,19 @@ index aad6e0c..2805a1e 100644 + Coroutine *co = qemu_coroutine_create(backup_start_savevm); + qemu_coroutine_enter(co, NULL); + } else { -+ backup_start_jobs(); ++ backup_start_jobs(); + } return g_strdup(backup_state.uuid_str); diff --git a/hmp.c b/hmp.c -index 9ac34c5..7b401cc 100644 +index b2c1f23..370cdf8 100644 --- a/hmp.c +++ b/hmp.c @@ -1052,7 +1052,8 @@ void hmp_backup(Monitor *mon, const QDict *qdict) Error *errp = NULL; - qmp_backup(backupfile, true, BACKUP_FORMAT_VMA, false, NULL, !!devlist, + qmp_backup(backup_file, true, BACKUP_FORMAT_VMA, false, NULL, !!devlist, - devlist, qdict_haskey(qdict, "speed"), speed, &errp); + devlist, qdict_haskey(qdict, "speed"), speed, false, false, + &errp); @@ -278,24 +278,25 @@ index 9ac34c5..7b401cc 100644 if (error_is_set(&errp)) { monitor_printf(mon, "%s\n", error_get_pretty(errp)); diff --git a/qapi-schema.json b/qapi-schema.json -index c91df47..53655e6 100644 +index 09ca8ef..1fabb67 100644 --- a/qapi-schema.json +++ b/qapi-schema.json -@@ -1882,13 +1882,15 @@ - # - # @speed: #optional the maximum speed, in bytes per second +@@ -1885,6 +1885,8 @@ + # @devlist: #optional list of block device names (separated by ',', ';' + # or ':'). By default the backup includes all writable block devices. # +# @state: #optional flag to include vm state +# # Returns: the uuid of the backup job # - # Since: 1.4.0 - ## - { 'command': 'backup', 'data': { 'backupfile': 'str', '*format': 'BackupFormat', -- '*config-filename': 'str', + # Since: 1.5.0 +@@ -1892,7 +1894,8 @@ + { 'command': 'backup', 'data': { 'backup-file': 'str', + '*format': 'BackupFormat', + '*config-file': 'str', - '*devlist': 'str', '*speed': 'int' }, -+ '*config-filename': 'str', '*devlist': 'str', -+ '*speed': 'int', '*state': 'bool' }, ++ '*devlist': 'str', '*speed': 'int', ++ '*state': 'bool' }, 'returns': 'str' } ## diff --git a/debian/patches/series b/debian/patches/series index 40fb0a7..da1cba2 100644 --- a/debian/patches/series +++ b/debian/patches/series @@ -10,11 +10,11 @@ internal-snapshot-async.patch qemu-img-convert-skipcreate-option.patch enable-kvm-by-default.patch # include qemu backup series -0001-RFC-Efficient-VM-backup-for-qemu.patch +0001-add-documenation-for-new-backup-framework.patch 0002-add-basic-backup-support-to-block-driver.patch 0003-add-backup-related-monitor-commands.patch 0004-introduce-new-vma-archive-format.patch 0005-add-regression-tests-for-backup.patch -#0006-add-vm-state-to-backups.patch +0006-add-vm-state-to-backups.patch # always-update-expected-downtime.patch virtio-balloon-fix-query.patch -- 2.39.2